kaslr.c 21 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2 3 4 5 6 7 8 9 10 11 12
/*
 * kaslr.c
 *
 * This contains the routines needed to generate a reasonable level of
 * entropy to choose a randomized kernel base address offset in support
 * of Kernel Address Space Layout Randomization (KASLR). Additionally
 * handles walking the physical memory maps (and tracking memory regions
 * to avoid) in order to select a physical memory location that can
 * contain the entire properly aligned running kernel image.
 *
 */
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31

/*
 * isspace() in linux/ctype.h is expected by next_args() to filter
 * out "space/lf/tab". While boot/ctype.h conflicts with linux/ctype.h,
 * since isdigit() is implemented in both of them. Hence disable it
 * here.
 */
#define BOOT_CTYPE_H

/*
 * _ctype[] in lib/ctype.c is needed by isspace() of linux/ctype.h.
 * While both lib/ctype.c and lib/cmdline.c will bring EXPORT_SYMBOL
 * which is meaningless and will cause compiling error in some cases.
 * So do not include linux/export.h and define EXPORT_SYMBOL(sym)
 * as empty.
 */
#define _LINUX_EXPORT_H
#define EXPORT_SYMBOL(sym)

32
#include "misc.h"
33
#include "error.h"
34
#include "../string.h"
35

36 37 38 39
#include <generated/compile.h>
#include <linux/module.h>
#include <linux/uts.h>
#include <linux/utsname.h>
40
#include <linux/ctype.h>
41
#include <linux/efi.h>
42
#include <generated/utsrelease.h>
43
#include <asm/efi.h>
44

45 46 47 48 49 50
/* Macros used by the included decompressor code below. */
#define STATIC
#include <linux/decompress/mm.h>

extern unsigned long get_cmd_line_ptr(void);

51
/* Simplified build-specific string for starting entropy. */
52
static const char build_str[] = UTS_RELEASE " (" LINUX_COMPILE_BY "@"
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70
		LINUX_COMPILE_HOST ") (" LINUX_COMPILER ") " UTS_VERSION;

static unsigned long rotate_xor(unsigned long hash, const void *area,
				size_t size)
{
	size_t i;
	unsigned long *ptr = (unsigned long *)area;

	for (i = 0; i < size / sizeof(hash); i++) {
		/* Rotate by odd number of bits and XOR. */
		hash = (hash << ((sizeof(hash) * 8) - 7)) | (hash >> 7);
		hash ^= ptr[i];
	}

	return hash;
}

/* Attempt to create a simple but unpredictable starting entropy. */
71
static unsigned long get_boot_seed(void)
72 73 74 75
{
	unsigned long hash = 0;

	hash = rotate_xor(hash, build_str, sizeof(build_str));
76
	hash = rotate_xor(hash, boot_params, sizeof(*boot_params));
77 78 79 80

	return hash;
}

81 82
#define KASLR_COMPRESSED_BOOT
#include "../../lib/kaslr.c"
83

84
struct mem_vector {
85 86
	unsigned long long start;
	unsigned long long size;
87 88
};

89 90 91 92 93
/* Only supporting at most 4 unusable memmap regions with kaslr */
#define MAX_MEMMAP_REGIONS	4

static bool memmap_too_large;

94

95 96 97 98
/* Store memory limit specified by "mem=nn[KMG]" or "memmap=nn[KMG]" */
unsigned long long mem_limit = ULLONG_MAX;


99 100 101 102 103
enum mem_avoid_index {
	MEM_AVOID_ZO_RANGE = 0,
	MEM_AVOID_INITRD,
	MEM_AVOID_CMDLINE,
	MEM_AVOID_BOOTPARAMS,
104 105
	MEM_AVOID_MEMMAP_BEGIN,
	MEM_AVOID_MEMMAP_END = MEM_AVOID_MEMMAP_BEGIN + MAX_MEMMAP_REGIONS - 1,
106 107 108
	MEM_AVOID_MAX,
};

109
static struct mem_vector mem_avoid[MEM_AVOID_MAX];
110 111 112 113 114 115 116 117 118 119 120 121

static bool mem_overlaps(struct mem_vector *one, struct mem_vector *two)
{
	/* Item one is entirely before item two. */
	if (one->start + one->size <= two->start)
		return false;
	/* Item one is entirely after item two. */
	if (one->start >= two->start + two->size)
		return false;
	return true;
}

122
char *skip_spaces(const char *str)
123
{
124 125 126
	while (isspace(*str))
		++str;
	return (char *)str;
127
}
128 129
#include "../../../../lib/ctype.c"
#include "../../../../lib/cmdline.c"
130 131 132 133 134 135 136 137 138 139 140 141 142 143

static int
parse_memmap(char *p, unsigned long long *start, unsigned long long *size)
{
	char *oldp;

	if (!p)
		return -EINVAL;

	/* We don't care about this option here */
	if (!strncmp(p, "exactmap", 8))
		return -EINVAL;

	oldp = p;
144
	*size = memparse(p, &p);
145 146 147 148 149 150 151
	if (p == oldp)
		return -EINVAL;

	switch (*p) {
	case '#':
	case '$':
	case '!':
152
		*start = memparse(p + 1, &p);
153
		return 0;
154 155 156 157 158 159 160 161 162 163 164
	case '@':
		/* memmap=nn@ss specifies usable region, should be skipped */
		*size = 0;
		/* Fall through */
	default:
		/*
		 * If w/o offset, only size specified, memmap=nn[KMG] has the
		 * same behaviour as mem=nn[KMG]. It limits the max address
		 * system can use. Region above the limit should be avoided.
		 */
		*start = 0;
165 166 167 168 169 170
		return 0;
	}

	return -EINVAL;
}

171
static void mem_avoid_memmap(char *str)
172
{
173
	static int i;
174

175
	if (i >= MAX_MEMMAP_REGIONS)
176 177 178 179 180 181 182 183 184 185 186 187 188 189
		return;

	while (str && (i < MAX_MEMMAP_REGIONS)) {
		int rc;
		unsigned long long start, size;
		char *k = strchr(str, ',');

		if (k)
			*k++ = 0;

		rc = parse_memmap(str, &start, &size);
		if (rc < 0)
			break;
		str = k;
190 191 192 193 194 195

		if (start == 0) {
			/* Store the specified memory limit if size > 0 */
			if (size > 0)
				mem_limit = size;

196
			continue;
197
		}
198 199 200 201 202 203 204 205 206 207 208

		mem_avoid[MEM_AVOID_MEMMAP_BEGIN + i].start = start;
		mem_avoid[MEM_AVOID_MEMMAP_BEGIN + i].size = size;
		i++;
	}

	/* More than 4 memmaps, fail kaslr */
	if ((i >= MAX_MEMMAP_REGIONS) && str)
		memmap_too_large = true;
}

209 210 211 212 213 214
static int handle_mem_memmap(void)
{
	char *args = (char *)get_cmd_line_ptr();
	size_t len = strlen((char *)args);
	char *tmp_cmdline;
	char *param, *val;
215
	u64 mem_size;
216

217
	if (!strstr(args, "memmap=") && !strstr(args, "mem="))
218 219 220
		return 0;

	tmp_cmdline = malloc(len + 1);
221
	if (!tmp_cmdline)
222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239
		error("Failed to allocate space for tmp_cmdline");

	memcpy(tmp_cmdline, args, len);
	tmp_cmdline[len] = 0;
	args = tmp_cmdline;

	/* Chew leading spaces */
	args = skip_spaces(args);

	while (*args) {
		args = next_arg(args, &param, &val);
		/* Stop at -- */
		if (!val && strcmp(param, "--") == 0) {
			warn("Only '--' specified in cmdline");
			free(tmp_cmdline);
			return -1;
		}

240
		if (!strcmp(param, "memmap")) {
241
			mem_avoid_memmap(val);
242 243 244 245 246 247 248 249 250 251 252 253
		} else if (!strcmp(param, "mem")) {
			char *p = val;

			if (!strcmp(p, "nopentium"))
				continue;
			mem_size = memparse(p, &p);
			if (mem_size == 0) {
				free(tmp_cmdline);
				return -EINVAL;
			}
			mem_limit = mem_size;
		}
254 255 256 257 258 259
	}

	free(tmp_cmdline);
	return 0;
}

260
/*
261 262 263
 * In theory, KASLR can put the kernel anywhere in the range of [16M, 64T).
 * The mem_avoid array is used to store the ranges that need to be avoided
 * when KASLR searches for an appropriate random address. We must avoid any
264
 * regions that are unsafe to overlap with during decompression, and other
265 266 267 268 269
 * things like the initrd, cmdline and boot_params. This comment seeks to
 * explain mem_avoid as clearly as possible since incorrect mem_avoid
 * memory ranges lead to really hard to debug boot failures.
 *
 * The initrd, cmdline, and boot_params are trivial to identify for
270
 * avoiding. They are MEM_AVOID_INITRD, MEM_AVOID_CMDLINE, and
271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
 * MEM_AVOID_BOOTPARAMS respectively below.
 *
 * What is not obvious how to avoid is the range of memory that is used
 * during decompression (MEM_AVOID_ZO_RANGE below). This range must cover
 * the compressed kernel (ZO) and its run space, which is used to extract
 * the uncompressed kernel (VO) and relocs.
 *
 * ZO's full run size sits against the end of the decompression buffer, so
 * we can calculate where text, data, bss, etc of ZO are positioned more
 * easily.
 *
 * For additional background, the decompression calculations can be found
 * in header.S, and the memory diagram is based on the one found in misc.c.
 *
 * The following conditions are already enforced by the image layouts and
 * associated code:
 *  - input + input_size >= output + output_size
 *  - kernel_total_size <= init_size
 *  - kernel_total_size <= output_size (see Note below)
 *  - output + init_size >= output + output_size
291
 *
292 293 294 295 296
 * (Note that kernel_total_size and output_size have no fundamental
 * relationship, but output_size is passed to choose_random_location
 * as a maximum of the two. The diagram is showing a case where
 * kernel_total_size is larger than output_size, but this case is
 * handled by bumping output_size.)
297
 *
298
 * The above conditions can be illustrated by a diagram:
299
 *
300 301 302 303 304 305 306
 * 0   output            input            input+input_size    output+init_size
 * |     |                 |                             |             |
 * |     |                 |                             |             |
 * |-----|--------|--------|--------------|-----------|--|-------------|
 *                |                       |           |
 *                |                       |           |
 * output+init_size-ZO_INIT_SIZE  output+output_size  output+kernel_total_size
307
 *
308 309
 * [output, output+init_size) is the entire memory range used for
 * extracting the compressed image.
310
 *
311 312
 * [output, output+kernel_total_size) is the range needed for the
 * uncompressed kernel (VO) and its run size (bss, brk, etc).
313
 *
314 315 316
 * [output, output+output_size) is VO plus relocs (i.e. the entire
 * uncompressed payload contained by ZO). This is the area of the buffer
 * written to during decompression.
317
 *
318 319 320
 * [output+init_size-ZO_INIT_SIZE, output+init_size) is the worst-case
 * range of the copied ZO and decompression code. (i.e. the range
 * covered backwards of size ZO_INIT_SIZE, starting from output+init_size.)
321
 *
322 323 324
 * [input, input+input_size) is the original copied compressed image (ZO)
 * (i.e. it does not include its run size). This range must be avoided
 * because it contains the data used for decompression.
325
 *
326 327 328
 * [input+input_size, output+init_size) is [_text, _end) for ZO. This
 * range includes ZO's heap and stack, and must be avoided since it
 * performs the decompression.
329
 *
330 331 332
 * Since the above two ranges need to be avoided and they are adjacent,
 * they can be merged, resulting in: [input, output+init_size) which
 * becomes the MEM_AVOID_ZO_RANGE below.
333
 */
334
static void mem_avoid_init(unsigned long input, unsigned long input_size,
335
			   unsigned long output)
336
{
337
	unsigned long init_size = boot_params->hdr.init_size;
338 339 340 341 342 343
	u64 initrd_start, initrd_size;
	u64 cmd_line, cmd_line_size;
	char *ptr;

	/*
	 * Avoid the region that is unsafe to overlap during
344
	 * decompression.
345
	 */
346 347
	mem_avoid[MEM_AVOID_ZO_RANGE].start = input;
	mem_avoid[MEM_AVOID_ZO_RANGE].size = (output + init_size) - input;
348 349
	add_identity_map(mem_avoid[MEM_AVOID_ZO_RANGE].start,
			 mem_avoid[MEM_AVOID_ZO_RANGE].size);
350 351

	/* Avoid initrd. */
352 353 354 355
	initrd_start  = (u64)boot_params->ext_ramdisk_image << 32;
	initrd_start |= boot_params->hdr.ramdisk_image;
	initrd_size  = (u64)boot_params->ext_ramdisk_size << 32;
	initrd_size |= boot_params->hdr.ramdisk_size;
356 357
	mem_avoid[MEM_AVOID_INITRD].start = initrd_start;
	mem_avoid[MEM_AVOID_INITRD].size = initrd_size;
358
	/* No need to set mapping for initrd, it will be handled in VO. */
359 360

	/* Avoid kernel command line. */
361 362
	cmd_line  = (u64)boot_params->ext_cmd_line_ptr << 32;
	cmd_line |= boot_params->hdr.cmd_line_ptr;
363 364
	/* Calculate size of cmd_line. */
	ptr = (char *)(unsigned long)cmd_line;
365
	for (cmd_line_size = 0; ptr[cmd_line_size++];)
366
		;
367 368
	mem_avoid[MEM_AVOID_CMDLINE].start = cmd_line;
	mem_avoid[MEM_AVOID_CMDLINE].size = cmd_line_size;
369 370
	add_identity_map(mem_avoid[MEM_AVOID_CMDLINE].start,
			 mem_avoid[MEM_AVOID_CMDLINE].size);
371

372 373 374
	/* Avoid boot parameters. */
	mem_avoid[MEM_AVOID_BOOTPARAMS].start = (unsigned long)boot_params;
	mem_avoid[MEM_AVOID_BOOTPARAMS].size = sizeof(*boot_params);
375 376 377 378 379
	add_identity_map(mem_avoid[MEM_AVOID_BOOTPARAMS].start,
			 mem_avoid[MEM_AVOID_BOOTPARAMS].size);

	/* We don't need to set a mapping for setup_data. */

380
	/* Mark the memmap regions we need to avoid */
381
	handle_mem_memmap();
382

383 384 385 386
#ifdef CONFIG_X86_VERBOSE_BOOTUP
	/* Make sure video RAM can be used. */
	add_identity_map(0, PMD_SIZE);
#endif
387 388
}

389 390 391 392 393 394
/*
 * Does this memory vector overlap a known avoided area? If so, record the
 * overlap region with the lowest address.
 */
static bool mem_avoid_overlap(struct mem_vector *img,
			      struct mem_vector *overlap)
395 396
{
	int i;
397
	struct setup_data *ptr;
398 399
	unsigned long earliest = img->start + img->size;
	bool is_overlapping = false;
400 401

	for (i = 0; i < MEM_AVOID_MAX; i++) {
402 403 404
		if (mem_overlaps(img, &mem_avoid[i]) &&
		    mem_avoid[i].start < earliest) {
			*overlap = mem_avoid[i];
405
			earliest = overlap->start;
406 407
			is_overlapping = true;
		}
408 409
	}

410
	/* Avoid all entries in the setup_data linked list. */
411
	ptr = (struct setup_data *)(unsigned long)boot_params->hdr.setup_data;
412 413 414
	while (ptr) {
		struct mem_vector avoid;

415
		avoid.start = (unsigned long)ptr;
416 417
		avoid.size = sizeof(*ptr) + ptr->len;

418 419
		if (mem_overlaps(img, &avoid) && (avoid.start < earliest)) {
			*overlap = avoid;
420
			earliest = overlap->start;
421 422
			is_overlapping = true;
		}
423 424 425 426

		ptr = (struct setup_data *)(unsigned long)ptr->next;
	}

427
	return is_overlapping;
428 429
}

430 431 432 433 434 435 436 437 438
struct slot_area {
	unsigned long addr;
	int num;
};

#define MAX_SLOT_AREA 100

static struct slot_area slot_areas[MAX_SLOT_AREA];

439
static unsigned long slot_max;
440

441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459
static unsigned long slot_area_index;

static void store_slot_info(struct mem_vector *region, unsigned long image_size)
{
	struct slot_area slot_area;

	if (slot_area_index == MAX_SLOT_AREA)
		return;

	slot_area.addr = region->start;
	slot_area.num = (region->size - image_size) /
			CONFIG_PHYSICAL_ALIGN + 1;

	if (slot_area.num > 0) {
		slot_areas[slot_area_index++] = slot_area;
		slot_max += slot_area.num;
	}
}

460 461
static unsigned long slots_fetch_random(void)
{
462 463 464
	unsigned long slot;
	int i;

465 466 467 468
	/* Handle case of no slots stored. */
	if (slot_max == 0)
		return 0;

469
	slot = kaslr_get_random_long("Physical") % slot_max;
470 471 472 473 474 475 476 477 478 479 480 481

	for (i = 0; i < slot_area_index; i++) {
		if (slot >= slot_areas[i].num) {
			slot -= slot_areas[i].num;
			continue;
		}
		return slot_areas[i].addr + slot * CONFIG_PHYSICAL_ALIGN;
	}

	if (i == slot_area_index)
		debug_putstr("slots_fetch_random() failed!?\n");
	return 0;
482 483
}

484
static void process_mem_region(struct mem_vector *entry,
485 486 487
			       unsigned long minimum,
			       unsigned long image_size)
{
488 489
	struct mem_vector region, overlap;
	struct slot_area slot_area;
490
	unsigned long start_orig, end;
491
	struct mem_vector cur_entry;
492

493
	/* On 32-bit, ignore entries entirely above our maximum. */
494
	if (IS_ENABLED(CONFIG_X86_32) && entry->start >= KERNEL_IMAGE_SIZE)
495 496 497
		return;

	/* Ignore entries entirely below our minimum. */
498
	if (entry->start + entry->size < minimum)
499 500
		return;

501
	/* Ignore entries above memory limit */
502 503
	end = min(entry->size + entry->start, mem_limit);
	if (entry->start >= end)
504
		return;
505 506
	cur_entry.start = entry->start;
	cur_entry.size = end - entry->start;
507

508
	region.start = cur_entry.start;
509
	region.size = cur_entry.size;
510

511 512 513
	/* Give up if slot area array is full. */
	while (slot_area_index < MAX_SLOT_AREA) {
		start_orig = region.start;
514

515 516 517
		/* Potentially raise address to minimum location. */
		if (region.start < minimum)
			region.start = minimum;
518

519 520
		/* Potentially raise address to meet alignment needs. */
		region.start = ALIGN(region.start, CONFIG_PHYSICAL_ALIGN);
521

522
		/* Did we raise the address above the passed in memory entry? */
523
		if (region.start > cur_entry.start + cur_entry.size)
524
			return;
525

526 527
		/* Reduce size by any delta from the original address. */
		region.size -= region.start - start_orig;
528

529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559
		/* On 32-bit, reduce region size to fit within max size. */
		if (IS_ENABLED(CONFIG_X86_32) &&
		    region.start + region.size > KERNEL_IMAGE_SIZE)
			region.size = KERNEL_IMAGE_SIZE - region.start;

		/* Return if region can't contain decompressed kernel */
		if (region.size < image_size)
			return;

		/* If nothing overlaps, store the region and return. */
		if (!mem_avoid_overlap(&region, &overlap)) {
			store_slot_info(&region, image_size);
			return;
		}

		/* Store beginning of region if holds at least image_size. */
		if (overlap.start > region.start + image_size) {
			struct mem_vector beginning;

			beginning.start = region.start;
			beginning.size = overlap.start - region.start;
			store_slot_info(&beginning, image_size);
		}

		/* Return if overlap extends to or past end of region. */
		if (overlap.start + overlap.size >= region.start + region.size)
			return;

		/* Clip off the overlapping region and start over. */
		region.size -= overlap.start - region.start + overlap.size;
		region.start = overlap.start + overlap.size;
560 561 562
	}
}

563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600
#ifdef CONFIG_EFI
/*
 * Returns true if mirror region found (and must have been processed
 * for slots adding)
 */
static bool
process_efi_entries(unsigned long minimum, unsigned long image_size)
{
	struct efi_info *e = &boot_params->efi_info;
	bool efi_mirror_found = false;
	struct mem_vector region;
	efi_memory_desc_t *md;
	unsigned long pmap;
	char *signature;
	u32 nr_desc;
	int i;

	signature = (char *)&e->efi_loader_signature;
	if (strncmp(signature, EFI32_LOADER_SIGNATURE, 4) &&
	    strncmp(signature, EFI64_LOADER_SIGNATURE, 4))
		return false;

#ifdef CONFIG_X86_32
	/* Can't handle data above 4GB at this time */
	if (e->efi_memmap_hi) {
		warn("EFI memmap is above 4GB, can't be handled now on x86_32. EFI should be disabled.\n");
		return false;
	}
	pmap =  e->efi_memmap;
#else
	pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
#endif

	nr_desc = e->efi_memmap_size / e->efi_memdesc_size;
	for (i = 0; i < nr_desc; i++) {
		md = efi_early_memdesc_ptr(pmap, e->efi_memdesc_size, i);
		if (md->attribute & EFI_MEMORY_MORE_RELIABLE) {
			efi_mirror_found = true;
601
			break;
602 603 604
		}
	}

605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634
	for (i = 0; i < nr_desc; i++) {
		md = efi_early_memdesc_ptr(pmap, e->efi_memdesc_size, i);

		/*
		 * Here we are more conservative in picking free memory than
		 * the EFI spec allows:
		 *
		 * According to the spec, EFI_BOOT_SERVICES_{CODE|DATA} are also
		 * free memory and thus available to place the kernel image into,
		 * but in practice there's firmware where using that memory leads
		 * to crashes.
		 *
		 * Only EFI_CONVENTIONAL_MEMORY is guaranteed to be free.
		 */
		if (md->type != EFI_CONVENTIONAL_MEMORY)
			continue;

		if (efi_mirror_found &&
		    !(md->attribute & EFI_MEMORY_MORE_RELIABLE))
			continue;

		region.start = md->phys_addr;
		region.size = md->num_pages << EFI_PAGE_SHIFT;
		process_mem_region(&region, minimum, image_size);
		if (slot_area_index == MAX_SLOT_AREA) {
			debug_putstr("Aborted EFI scan (slot_areas full)!\n");
			break;
		}
	}
	return true;
635 636 637 638 639 640 641 642 643
}
#else
static inline bool
process_efi_entries(unsigned long minimum, unsigned long image_size)
{
	return false;
}
#endif

644 645
static void process_e820_entries(unsigned long minimum,
				 unsigned long image_size)
646 647
{
	int i;
648
	struct mem_vector region;
649 650 651 652 653 654 655 656
	struct boot_e820_entry *entry;

	/* Verify potential e820 positions, appending to slots list. */
	for (i = 0; i < boot_params->e820_entries; i++) {
		entry = &boot_params->e820_table[i];
		/* Skip non-RAM entries. */
		if (entry->type != E820_TYPE_RAM)
			continue;
657 658
		region.start = entry->addr;
		region.size = entry->size;
659
		process_mem_region(&region, minimum, image_size);
660 661 662 663 664 665
		if (slot_area_index == MAX_SLOT_AREA) {
			debug_putstr("Aborted e820 scan (slot_areas full)!\n");
			break;
		}
	}
}
666

667 668 669
static unsigned long find_random_phys_addr(unsigned long minimum,
					   unsigned long image_size)
{
670 671
	/* Check if we had too many memmaps. */
	if (memmap_too_large) {
672
		debug_putstr("Aborted memory entries scan (more than 4 memmap= args)!\n");
673 674 675
		return 0;
	}

676 677 678
	/* Make sure minimum is aligned. */
	minimum = ALIGN(minimum, CONFIG_PHYSICAL_ALIGN);

679 680 681
	if (process_efi_entries(minimum, image_size))
		return slots_fetch_random();

682
	process_e820_entries(minimum, image_size);
683 684 685
	return slots_fetch_random();
}

686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703
static unsigned long find_random_virt_addr(unsigned long minimum,
					   unsigned long image_size)
{
	unsigned long slots, random_addr;

	/* Make sure minimum is aligned. */
	minimum = ALIGN(minimum, CONFIG_PHYSICAL_ALIGN);
	/* Align image_size for easy slot calculations. */
	image_size = ALIGN(image_size, CONFIG_PHYSICAL_ALIGN);

	/*
	 * There are how many CONFIG_PHYSICAL_ALIGN-sized slots
	 * that can hold image_size within the range of minimum to
	 * KERNEL_IMAGE_SIZE?
	 */
	slots = (KERNEL_IMAGE_SIZE - minimum - image_size) /
		 CONFIG_PHYSICAL_ALIGN + 1;

704
	random_addr = kaslr_get_random_long("Virtual") % slots;
705 706 707 708

	return random_addr * CONFIG_PHYSICAL_ALIGN + minimum;
}

709 710 711 712
/*
 * Since this function examines addresses much more numerically,
 * it takes the input and output pointers as 'unsigned long'.
 */
713 714 715 716 717
void choose_random_location(unsigned long input,
			    unsigned long input_size,
			    unsigned long *output,
			    unsigned long output_size,
			    unsigned long *virt_addr)
718
{
719
	unsigned long random_addr, min_addr;
720 721

	if (cmdline_find_option_bool("nokaslr")) {
722
		warn("KASLR disabled: 'nokaslr' on cmdline.");
723
		return;
724 725
	}

726
	boot_params->hdr.loadflags |= KASLR_FLAG;
727

728 729 730
	/* Prepare to add new identity pagetables on demand. */
	initialize_identity_maps();

731
	/* Record the various known unsafe memory ranges. */
732
	mem_avoid_init(input, input_size, *output);
733

734 735 736 737 738 739 740
	/*
	 * Low end of the randomization range should be the
	 * smaller of 512M or the initial kernel image
	 * location:
	 */
	min_addr = min(*output, 512UL << 20);

741
	/* Walk available memory entries to find a random address. */
742
	random_addr = find_random_phys_addr(min_addr, output_size);
743
	if (!random_addr) {
744
		warn("Physical KASLR disabled: no suitable memory region!");
745 746 747 748 749 750
	} else {
		/* Update the new physical address location. */
		if (*output != random_addr) {
			add_identity_map(random_addr, output_size);
			*output = random_addr;
		}
751 752 753 754 755 756 757 758 759

		/*
		 * This loads the identity mapping page table.
		 * This should only be done if a new physical address
		 * is found for the kernel, otherwise we should keep
		 * the old page table to make it be like the "nokaslr"
		 * case.
		 */
		finalize_identity_maps();
760 761
	}

762 763 764 765 766

	/* Pick random virtual address starting from LOAD_PHYSICAL_ADDR. */
	if (IS_ENABLED(CONFIG_X86_64))
		random_addr = find_random_virt_addr(LOAD_PHYSICAL_ADDR, output_size);
	*virt_addr = random_addr;
767
}