Commit 686140a1 authored by Vasily Gorbik's avatar Vasily Gorbik Committed by Martin Schwidefsky
Browse files

s390: introduce CPU alternatives



Implement CPU alternatives, which allows to optionally patch newer
instructions at runtime, based on CPU facilities availability.

A new kernel boot parameter "noaltinstr" disables patching.

Current implementation is derived from x86 alternatives. Although
ideal instructions padding (when altinstr is longer then oldinstr)
is added at compile time, and no oldinstr nops optimization has to be
done at runtime. Also couple of compile time sanity checks are done:
1. oldinstr and altinstr must be <= 254 bytes long,
2. oldinstr and altinstr must not have an odd length.

alternative(oldinstr, altinstr, facility);
alternative_2(oldinstr, altinstr1, facility1, altinstr2, facility2);

Both compile time and runtime padding consists of either 6/4/2 bytes nop
or a jump (brcl) + 2 bytes nop filler if padding is longer then 6 bytes.

.altinstructions and .altinstr_replacement sections are part of
__init_begin : __init_end region and are freed after initialization.
Signed-off-by: default avatarVasily Gorbik <gor@linux.vnet.ibm.com>
Signed-off-by: default avatarMartin Schwidefsky <schwidefsky@de.ibm.com>
parent 76b3b62a
......@@ -2548,6 +2548,9 @@
noalign [KNL,ARM]
noaltinstr [S390] Disables alternative instructions patching
(CPU alternatives feature).
noapic [SMP,APIC] Tells the kernel to not make use of any
IOAPICs that may be present in the system.
......
......@@ -537,6 +537,22 @@ config ARCH_RANDOM
If unsure, say Y.
config ALTERNATIVES
def_bool y
prompt "Patch optimized instructions for running CPU type"
help
When enabled the kernel code is compiled with additional
alternative instructions blocks optimized for newer CPU types.
These alternative instructions blocks are patched at kernel boot
time when running CPU supports them. This mechanism is used to
optimize some critical code paths (i.e. spinlocks) for newer CPUs
even if kernel is build to support older machine generations.
This mechanism could be disabled by appending "noaltinstr"
option to the kernel command line.
If unsure, say Y.
endmenu
menu "Memory setup"
......@@ -811,6 +827,7 @@ config PFAULT
config SHARED_KERNEL
bool "VM shared kernel support"
depends on !JUMP_LABEL
depends on !ALTERNATIVES
help
Select this option, if you want to share the text segment of the
Linux kernel between different VM guests. This reduces memory
......
#ifndef _ASM_S390_ALTERNATIVE_H
#define _ASM_S390_ALTERNATIVE_H
#ifndef __ASSEMBLY__
#include <linux/types.h>
#include <linux/stddef.h>
#include <linux/stringify.h>
struct alt_instr {
s32 instr_offset; /* original instruction */
s32 repl_offset; /* offset to replacement instruction */
u16 facility; /* facility bit set for replacement */
u8 instrlen; /* length of original instruction */
u8 replacementlen; /* length of new instruction */
} __packed;
#ifdef CONFIG_ALTERNATIVES
extern void apply_alternative_instructions(void);
extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
#else
static inline void apply_alternative_instructions(void) {};
static inline void apply_alternatives(struct alt_instr *start,
struct alt_instr *end) {};
#endif
/*
* |661: |662: |6620 |663:
* +-----------+---------------------+
* | oldinstr | oldinstr_padding |
* | +----------+----------+
* | | | |
* | | >6 bytes |6/4/2 nops|
* | |6 bytes jg----------->
* +-----------+---------------------+
* ^^ static padding ^^
*
* .altinstr_replacement section
* +---------------------+-----------+
* |6641: |6651:
* | alternative instr 1 |
* +-----------+---------+- - - - - -+
* |6642: |6652: |
* | alternative instr 2 | padding
* +---------------------+- - - - - -+
* ^ runtime ^
*
* .altinstructions section
* +---------------------------------+
* | alt_instr entries for each |
* | alternative instr |
* +---------------------------------+
*/
#define b_altinstr(num) "664"#num
#define e_altinstr(num) "665"#num
#define e_oldinstr_pad_end "663"
#define oldinstr_len "662b-661b"
#define oldinstr_total_len e_oldinstr_pad_end"b-661b"
#define altinstr_len(num) e_altinstr(num)"b-"b_altinstr(num)"b"
#define oldinstr_pad_len(num) \
"-(((" altinstr_len(num) ")-(" oldinstr_len ")) > 0) * " \
"((" altinstr_len(num) ")-(" oldinstr_len "))"
#define INSTR_LEN_SANITY_CHECK(len) \
".if " len " > 254\n" \
"\t.error \"cpu alternatives does not support instructions " \
"blocks > 254 bytes\"\n" \
".endif\n" \
".if (" len ") %% 2\n" \
"\t.error \"cpu alternatives instructions length is odd\"\n" \
".endif\n"
#define OLDINSTR_PADDING(oldinstr, num) \
".if " oldinstr_pad_len(num) " > 6\n" \
"\tjg " e_oldinstr_pad_end "f\n" \
"6620:\n" \
"\t.fill (" oldinstr_pad_len(num) " - (6620b-662b)) / 2, 2, 0x0700\n" \
".else\n" \
"\t.fill " oldinstr_pad_len(num) " / 6, 6, 0xc0040000\n" \
"\t.fill " oldinstr_pad_len(num) " %% 6 / 4, 4, 0x47000000\n" \
"\t.fill " oldinstr_pad_len(num) " %% 6 %% 4 / 2, 2, 0x0700\n" \
".endif\n"
#define OLDINSTR(oldinstr, num) \
"661:\n\t" oldinstr "\n662:\n" \
OLDINSTR_PADDING(oldinstr, num) \
e_oldinstr_pad_end ":\n" \
INSTR_LEN_SANITY_CHECK(oldinstr_len)
#define OLDINSTR_2(oldinstr, num1, num2) \
"661:\n\t" oldinstr "\n662:\n" \
".if " altinstr_len(num1) " < " altinstr_len(num2) "\n" \
OLDINSTR_PADDING(oldinstr, num2) \
".else\n" \
OLDINSTR_PADDING(oldinstr, num1) \
".endif\n" \
e_oldinstr_pad_end ":\n" \
INSTR_LEN_SANITY_CHECK(oldinstr_len)
#define ALTINSTR_ENTRY(facility, num) \
"\t.long 661b - .\n" /* old instruction */ \
"\t.long " b_altinstr(num)"b - .\n" /* alt instruction */ \
"\t.word " __stringify(facility) "\n" /* facility bit */ \
"\t.byte " oldinstr_total_len "\n" /* source len */ \
"\t.byte " altinstr_len(num) "\n" /* alt instruction len */
#define ALTINSTR_REPLACEMENT(altinstr, num) /* replacement */ \
b_altinstr(num)":\n\t" altinstr "\n" e_altinstr(num) ":\n" \
INSTR_LEN_SANITY_CHECK(altinstr_len(num))
#ifdef CONFIG_ALTERNATIVES
/* alternative assembly primitive: */
#define ALTERNATIVE(oldinstr, altinstr, facility) \
".pushsection .altinstr_replacement, \"ax\"\n" \
ALTINSTR_REPLACEMENT(altinstr, 1) \
".popsection\n" \
OLDINSTR(oldinstr, 1) \
".pushsection .altinstructions,\"a\"\n" \
ALTINSTR_ENTRY(facility, 1) \
".popsection\n"
#define ALTERNATIVE_2(oldinstr, altinstr1, facility1, altinstr2, facility2)\
".pushsection .altinstr_replacement, \"ax\"\n" \
ALTINSTR_REPLACEMENT(altinstr1, 1) \
ALTINSTR_REPLACEMENT(altinstr2, 2) \
".popsection\n" \
OLDINSTR_2(oldinstr, 1, 2) \
".pushsection .altinstructions,\"a\"\n" \
ALTINSTR_ENTRY(facility1, 1) \
ALTINSTR_ENTRY(facility2, 2) \
".popsection\n"
#else
/* Alternative instructions are disabled, let's put just oldinstr in */
#define ALTERNATIVE(oldinstr, altinstr, facility) \
oldinstr "\n"
#define ALTERNATIVE_2(oldinstr, altinstr1, facility1, altinstr2, facility2) \
oldinstr "\n"
#endif
/*
* Alternative instructions for different CPU types or capabilities.
*
* This allows to use optimized instructions even on generic binary
* kernels.
*
* oldinstr is padded with jump and nops at compile time if altinstr is
* longer. altinstr is padded with jump and nops at run-time during patching.
*
* For non barrier like inlines please define new variants
* without volatile and memory clobber.
*/
#define alternative(oldinstr, altinstr, facility) \
asm volatile(ALTERNATIVE(oldinstr, altinstr, facility) : : : "memory")
#define alternative_2(oldinstr, altinstr1, facility1, altinstr2, facility2) \
asm volatile(ALTERNATIVE_2(oldinstr, altinstr1, facility1, \
altinstr2, facility2) ::: "memory")
#endif /* __ASSEMBLY__ */
#endif /* _ASM_S390_ALTERNATIVE_H */
......@@ -76,6 +76,7 @@ obj-$(CONFIG_KPROBES) += kprobes.o
obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o
obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
obj-$(CONFIG_UPROBES) += uprobes.o
obj-$(CONFIG_ALTERNATIVES) += alternative.o
obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_cpum_cf.o perf_cpum_sf.o
obj-$(CONFIG_PERF_EVENTS) += perf_cpum_cf_events.o
......
#include <linux/module.h>
#include <asm/alternative.h>
#include <asm/facility.h>
#define MAX_PATCH_LEN (255 - 1)
static int __initdata_or_module alt_instr_disabled;
static int __init disable_alternative_instructions(char *str)
{
alt_instr_disabled = 1;
return 0;
}
early_param("noaltinstr", disable_alternative_instructions);
struct brcl_insn {
u16 opc;
s32 disp;
} __packed;
static u16 __initdata_or_module nop16 = 0x0700;
static u32 __initdata_or_module nop32 = 0x47000000;
static struct brcl_insn __initdata_or_module nop48 = {
0xc004, 0
};
static const void *nops[] __initdata_or_module = {
&nop16,
&nop32,
&nop48
};
static void __init_or_module add_jump_padding(void *insns, unsigned int len)
{
struct brcl_insn brcl = {
0xc0f4,
len / 2
};
memcpy(insns, &brcl, sizeof(brcl));
insns += sizeof(brcl);
len -= sizeof(brcl);
while (len > 0) {
memcpy(insns, &nop16, 2);
insns += 2;
len -= 2;
}
}
static void __init_or_module add_padding(void *insns, unsigned int len)
{
if (len > 6)
add_jump_padding(insns, len);
else if (len >= 2)
memcpy(insns, nops[len / 2 - 1], len);
}
static void __init_or_module __apply_alternatives(struct alt_instr *start,
struct alt_instr *end)
{
struct alt_instr *a;
u8 *instr, *replacement;
u8 insnbuf[MAX_PATCH_LEN];
/*
* The scan order should be from start to end. A later scanned
* alternative code can overwrite previously scanned alternative code.
*/
for (a = start; a < end; a++) {
int insnbuf_sz = 0;
instr = (u8 *)&a->instr_offset + a->instr_offset;
replacement = (u8 *)&a->repl_offset + a->repl_offset;
if (!test_facility(a->facility))
continue;
if (unlikely(a->instrlen % 2 || a->replacementlen % 2)) {
WARN_ONCE(1, "cpu alternatives instructions length is "
"odd, skipping patching\n");
continue;
}
memcpy(insnbuf, replacement, a->replacementlen);
insnbuf_sz = a->replacementlen;
if (a->instrlen > a->replacementlen) {
add_padding(insnbuf + a->replacementlen,
a->instrlen - a->replacementlen);
insnbuf_sz += a->instrlen - a->replacementlen;
}
s390_kernel_write(instr, insnbuf, insnbuf_sz);
}
}
void __init_or_module apply_alternatives(struct alt_instr *start,
struct alt_instr *end)
{
if (!alt_instr_disabled)
__apply_alternatives(start, end);
}
extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
void __init apply_alternative_instructions(void)
{
apply_alternatives(__alt_instructions, __alt_instructions_end);
}
......@@ -31,6 +31,7 @@
#include <linux/kernel.h>
#include <linux/moduleloader.h>
#include <linux/bug.h>
#include <asm/alternative.h>
#if 0
#define DEBUGP printk
......@@ -429,6 +430,22 @@ int module_finalize(const Elf_Ehdr *hdr,
const Elf_Shdr *sechdrs,
struct module *me)
{
const Elf_Shdr *s;
char *secstrings;
if (IS_ENABLED(CONFIG_ALTERNATIVES)) {
secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
if (!strcmp(".altinstructions",
secstrings + s->sh_name)) {
/* patch .altinstructions */
void *aseg = (void *)s->sh_addr;
apply_alternatives(aseg, aseg + s->sh_size);
}
}
}
jump_label_apply_nops(me);
return 0;
}
......@@ -66,6 +66,7 @@
#include <asm/sclp.h>
#include <asm/sysinfo.h>
#include <asm/numa.h>
#include <asm/alternative.h>
#include "entry.h"
/*
......@@ -957,6 +958,8 @@ void __init setup_arch(char **cmdline_p)
conmode_default();
set_preferred_console();
apply_alternative_instructions();
/* Setup zfcpdump support */
setup_zfcpdump();
......
......@@ -104,6 +104,29 @@ SECTIONS
EXIT_DATA
}
/*
* struct alt_inst entries. From the header (alternative.h):
* "Alternative instructions for different CPU types or capabilities"
* Think locking instructions on spinlocks.
* Note, that it is a part of __init region.
*/
. = ALIGN(8);
.altinstructions : {
__alt_instructions = .;
*(.altinstructions)
__alt_instructions_end = .;
}
/*
* And here are the replacement instructions. The linker sticks
* them as binary blobs. The .altinstructions has enough data to
* get the address and the length of them to patch the kernel safely.
* Note, that it is a part of __init region.
*/
.altinstr_replacement : {
*(.altinstr_replacement)
}
/* early.c uses stsi, which requires page aligned data. */
. = ALIGN(PAGE_SIZE);
INIT_DATA_SECTION(0x100)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment