Commit e192be9d authored by Theodore Ts'o's avatar Theodore Ts'o
Browse files

random: replace non-blocking pool with a Chacha20-based CRNG



The CRNG is faster, and we don't pretend to track entropy usage in the
CRNG any more.
Signed-off-by: default avatarTheodore Ts'o <tytso@mit.edu>
parent b1132dea
...@@ -15,72 +15,11 @@ ...@@ -15,72 +15,11 @@
#include <linux/module.h> #include <linux/module.h>
#include <crypto/chacha20.h> #include <crypto/chacha20.h>
static inline u32 rotl32(u32 v, u8 n)
{
return (v << n) | (v >> (sizeof(v) * 8 - n));
}
static inline u32 le32_to_cpuvp(const void *p) static inline u32 le32_to_cpuvp(const void *p)
{ {
return le32_to_cpup(p); return le32_to_cpup(p);
} }
static void chacha20_block(u32 *state, void *stream)
{
u32 x[16], *out = stream;
int i;
for (i = 0; i < ARRAY_SIZE(x); i++)
x[i] = state[i];
for (i = 0; i < 20; i += 2) {
x[0] += x[4]; x[12] = rotl32(x[12] ^ x[0], 16);
x[1] += x[5]; x[13] = rotl32(x[13] ^ x[1], 16);
x[2] += x[6]; x[14] = rotl32(x[14] ^ x[2], 16);
x[3] += x[7]; x[15] = rotl32(x[15] ^ x[3], 16);
x[8] += x[12]; x[4] = rotl32(x[4] ^ x[8], 12);
x[9] += x[13]; x[5] = rotl32(x[5] ^ x[9], 12);
x[10] += x[14]; x[6] = rotl32(x[6] ^ x[10], 12);
x[11] += x[15]; x[7] = rotl32(x[7] ^ x[11], 12);
x[0] += x[4]; x[12] = rotl32(x[12] ^ x[0], 8);
x[1] += x[5]; x[13] = rotl32(x[13] ^ x[1], 8);
x[2] += x[6]; x[14] = rotl32(x[14] ^ x[2], 8);
x[3] += x[7]; x[15] = rotl32(x[15] ^ x[3], 8);
x[8] += x[12]; x[4] = rotl32(x[4] ^ x[8], 7);
x[9] += x[13]; x[5] = rotl32(x[5] ^ x[9], 7);
x[10] += x[14]; x[6] = rotl32(x[6] ^ x[10], 7);
x[11] += x[15]; x[7] = rotl32(x[7] ^ x[11], 7);
x[0] += x[5]; x[15] = rotl32(x[15] ^ x[0], 16);
x[1] += x[6]; x[12] = rotl32(x[12] ^ x[1], 16);
x[2] += x[7]; x[13] = rotl32(x[13] ^ x[2], 16);
x[3] += x[4]; x[14] = rotl32(x[14] ^ x[3], 16);
x[10] += x[15]; x[5] = rotl32(x[5] ^ x[10], 12);
x[11] += x[12]; x[6] = rotl32(x[6] ^ x[11], 12);
x[8] += x[13]; x[7] = rotl32(x[7] ^ x[8], 12);
x[9] += x[14]; x[4] = rotl32(x[4] ^ x[9], 12);
x[0] += x[5]; x[15] = rotl32(x[15] ^ x[0], 8);
x[1] += x[6]; x[12] = rotl32(x[12] ^ x[1], 8);
x[2] += x[7]; x[13] = rotl32(x[13] ^ x[2], 8);
x[3] += x[4]; x[14] = rotl32(x[14] ^ x[3], 8);
x[10] += x[15]; x[5] = rotl32(x[5] ^ x[10], 7);
x[11] += x[12]; x[6] = rotl32(x[6] ^ x[11], 7);
x[8] += x[13]; x[7] = rotl32(x[7] ^ x[8], 7);
x[9] += x[14]; x[4] = rotl32(x[4] ^ x[9], 7);
}
for (i = 0; i < ARRAY_SIZE(x); i++)
out[i] = cpu_to_le32(x[i] + state[i]);
state[12]++;
}
static void chacha20_docrypt(u32 *state, u8 *dst, const u8 *src, static void chacha20_docrypt(u32 *state, u8 *dst, const u8 *src,
unsigned int bytes) unsigned int bytes)
{ {
......
...@@ -261,6 +261,7 @@ ...@@ -261,6 +261,7 @@
#include <linux/syscalls.h> #include <linux/syscalls.h>
#include <linux/completion.h> #include <linux/completion.h>
#include <linux/uuid.h> #include <linux/uuid.h>
#include <crypto/chacha20.h>
#include <asm/processor.h> #include <asm/processor.h>
#include <asm/uaccess.h> #include <asm/uaccess.h>
...@@ -413,6 +414,31 @@ static struct fasync_struct *fasync; ...@@ -413,6 +414,31 @@ static struct fasync_struct *fasync;
static DEFINE_SPINLOCK(random_ready_list_lock); static DEFINE_SPINLOCK(random_ready_list_lock);
static LIST_HEAD(random_ready_list); static LIST_HEAD(random_ready_list);
struct crng_state {
__u32 state[16];
unsigned long init_time;
spinlock_t lock;
};
struct crng_state primary_crng = {
.lock = __SPIN_LOCK_UNLOCKED(primary_crng.lock),
};
/*
* crng_init = 0 --> Uninitialized
* 1 --> Initialized
* 2 --> Initialized from input_pool
*
* crng_init is protected by primary_crng->lock, and only increases
* its value (from 0->1->2).
*/
static int crng_init = 0;
#define crng_ready() (likely(crng_init > 0))
static int crng_init_cnt = 0;
#define CRNG_INIT_CNT_THRESH (2*CHACHA20_KEY_SIZE)
static void extract_crng(__u8 out[CHACHA20_BLOCK_SIZE]);
static void process_random_ready_list(void);
/********************************************************************** /**********************************************************************
* *
* OS independent entropy store. Here are the functions which handle * OS independent entropy store. Here are the functions which handle
...@@ -442,10 +468,15 @@ struct entropy_store { ...@@ -442,10 +468,15 @@ struct entropy_store {
__u8 last_data[EXTRACT_SIZE]; __u8 last_data[EXTRACT_SIZE];
}; };
static ssize_t extract_entropy(struct entropy_store *r, void *buf,
size_t nbytes, int min, int rsvd);
static ssize_t _extract_entropy(struct entropy_store *r, void *buf,
size_t nbytes, int fips);
static void crng_reseed(struct crng_state *crng, struct entropy_store *r);
static void push_to_pool(struct work_struct *work); static void push_to_pool(struct work_struct *work);
static __u32 input_pool_data[INPUT_POOL_WORDS]; static __u32 input_pool_data[INPUT_POOL_WORDS];
static __u32 blocking_pool_data[OUTPUT_POOL_WORDS]; static __u32 blocking_pool_data[OUTPUT_POOL_WORDS];
static __u32 nonblocking_pool_data[OUTPUT_POOL_WORDS];
static struct entropy_store input_pool = { static struct entropy_store input_pool = {
.poolinfo = &poolinfo_table[0], .poolinfo = &poolinfo_table[0],
...@@ -466,16 +497,6 @@ static struct entropy_store blocking_pool = { ...@@ -466,16 +497,6 @@ static struct entropy_store blocking_pool = {
push_to_pool), push_to_pool),
}; };
static struct entropy_store nonblocking_pool = {
.poolinfo = &poolinfo_table[1],
.name = "nonblocking",
.pull = &input_pool,
.lock = __SPIN_LOCK_UNLOCKED(nonblocking_pool.lock),
.pool = nonblocking_pool_data,
.push_work = __WORK_INITIALIZER(nonblocking_pool.push_work,
push_to_pool),
};
static __u32 const twist_table[8] = { static __u32 const twist_table[8] = {
0x00000000, 0x3b6e20c8, 0x76dc4190, 0x4db26158, 0x00000000, 0x3b6e20c8, 0x76dc4190, 0x4db26158,
0xedb88320, 0xd6d6a3e8, 0x9b64c2b0, 0xa00ae278 }; 0xedb88320, 0xd6d6a3e8, 0x9b64c2b0, 0xa00ae278 };
...@@ -678,12 +699,6 @@ static void credit_entropy_bits(struct entropy_store *r, int nbits) ...@@ -678,12 +699,6 @@ static void credit_entropy_bits(struct entropy_store *r, int nbits)
if (!r->initialized && r->entropy_total > 128) { if (!r->initialized && r->entropy_total > 128) {
r->initialized = 1; r->initialized = 1;
r->entropy_total = 0; r->entropy_total = 0;
if (r == &nonblocking_pool) {
prandom_reseed_late();
process_random_ready_list();
wake_up_all(&urandom_init_wait);
pr_notice("random: %s pool is initialized\n", r->name);
}
} }
trace_credit_entropy_bits(r->name, nbits, trace_credit_entropy_bits(r->name, nbits,
...@@ -693,30 +708,27 @@ static void credit_entropy_bits(struct entropy_store *r, int nbits) ...@@ -693,30 +708,27 @@ static void credit_entropy_bits(struct entropy_store *r, int nbits)
if (r == &input_pool) { if (r == &input_pool) {
int entropy_bits = entropy_count >> ENTROPY_SHIFT; int entropy_bits = entropy_count >> ENTROPY_SHIFT;
if (crng_init < 2 && entropy_bits >= 128) {
crng_reseed(&primary_crng, r);
entropy_bits = r->entropy_count >> ENTROPY_SHIFT;
}
/* should we wake readers? */ /* should we wake readers? */
if (entropy_bits >= random_read_wakeup_bits) { if (entropy_bits >= random_read_wakeup_bits) {
wake_up_interruptible(&random_read_wait); wake_up_interruptible(&random_read_wait);
kill_fasync(&fasync, SIGIO, POLL_IN); kill_fasync(&fasync, SIGIO, POLL_IN);
} }
/* If the input pool is getting full, send some /* If the input pool is getting full, send some
* entropy to the two output pools, flipping back and * entropy to the blocking pool until it is 75% full.
* forth between them, until the output pools are 75%
* full.
*/ */
if (entropy_bits > random_write_wakeup_bits && if (entropy_bits > random_write_wakeup_bits &&
r->initialized && r->initialized &&
r->entropy_total >= 2*random_read_wakeup_bits) { r->entropy_total >= 2*random_read_wakeup_bits) {
static struct entropy_store *last = &blocking_pool;
struct entropy_store *other = &blocking_pool; struct entropy_store *other = &blocking_pool;
if (last == &blocking_pool)
other = &nonblocking_pool;
if (other->entropy_count <= if (other->entropy_count <=
3 * other->poolinfo->poolfracbits / 4) 3 * other->poolinfo->poolfracbits / 4) {
last = other; schedule_work(&other->push_work);
if (last->entropy_count <=
3 * last->poolinfo->poolfracbits / 4) {
schedule_work(&last->push_work);
r->entropy_total = 0; r->entropy_total = 0;
} }
} }
...@@ -734,6 +746,152 @@ static void credit_entropy_bits_safe(struct entropy_store *r, int nbits) ...@@ -734,6 +746,152 @@ static void credit_entropy_bits_safe(struct entropy_store *r, int nbits)
credit_entropy_bits(r, nbits); credit_entropy_bits(r, nbits);
} }
/*********************************************************************
*
* CRNG using CHACHA20
*
*********************************************************************/
#define CRNG_RESEED_INTERVAL (300*HZ)
static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait);
static void crng_initialize(struct crng_state *crng)
{
int i;
unsigned long rv;
memcpy(&crng->state[0], "expand 32-byte k", 16);
if (crng == &primary_crng)
_extract_entropy(&input_pool, &crng->state[4],
sizeof(__u32) * 12, 0);
else
get_random_bytes(&crng->state[4], sizeof(__u32) * 12);
for (i = 4; i < 16; i++) {
if (!arch_get_random_seed_long(&rv) &&
!arch_get_random_long(&rv))
rv = random_get_entropy();
crng->state[i] ^= rv;
}
crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1;
}
static int crng_fast_load(const char *cp, size_t len)
{
unsigned long flags;
char *p;
if (!spin_trylock_irqsave(&primary_crng.lock, flags))
return 0;
if (crng_ready()) {
spin_unlock_irqrestore(&primary_crng.lock, flags);
return 0;
}
p = (unsigned char *) &primary_crng.state[4];
while (len > 0 && crng_init_cnt < CRNG_INIT_CNT_THRESH) {
p[crng_init_cnt % CHACHA20_KEY_SIZE] ^= *cp;
cp++; crng_init_cnt++; len--;
}
if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) {
crng_init = 1;
wake_up_interruptible(&crng_init_wait);
pr_notice("random: fast init done\n");
}
spin_unlock_irqrestore(&primary_crng.lock, flags);
return 1;
}
static void crng_reseed(struct crng_state *crng, struct entropy_store *r)
{
unsigned long flags;
int i, num;
union {
__u8 block[CHACHA20_BLOCK_SIZE];
__u32 key[8];
} buf;
if (r) {
num = extract_entropy(r, &buf, 32, 16, 0);
if (num == 0)
return;
} else
extract_crng(buf.block);
spin_lock_irqsave(&primary_crng.lock, flags);
for (i = 0; i < 8; i++) {
unsigned long rv;
if (!arch_get_random_seed_long(&rv) &&
!arch_get_random_long(&rv))
rv = random_get_entropy();
crng->state[i+4] ^= buf.key[i] ^ rv;
}
memzero_explicit(&buf, sizeof(buf));
crng->init_time = jiffies;
if (crng == &primary_crng && crng_init < 2) {
crng_init = 2;
process_random_ready_list();
wake_up_interruptible(&crng_init_wait);
pr_notice("random: crng init done\n");
}
spin_unlock_irqrestore(&primary_crng.lock, flags);
}
static inline void crng_wait_ready(void)
{
wait_event_interruptible(crng_init_wait, crng_ready());
}
static void extract_crng(__u8 out[CHACHA20_BLOCK_SIZE])
{
unsigned long v, flags;
struct crng_state *crng = &primary_crng;
if (crng_init > 1 &&
time_after(jiffies, crng->init_time + CRNG_RESEED_INTERVAL))
crng_reseed(crng, &input_pool);
spin_lock_irqsave(&crng->lock, flags);
if (arch_get_random_long(&v))
crng->state[14] ^= v;
chacha20_block(&crng->state[0], out);
if (crng->state[12] == 0)
crng->state[13]++;
spin_unlock_irqrestore(&crng->lock, flags);
}
static ssize_t extract_crng_user(void __user *buf, size_t nbytes)
{
ssize_t ret = 0, i;
__u8 tmp[CHACHA20_BLOCK_SIZE];
int large_request = (nbytes > 256);
while (nbytes) {
if (large_request && need_resched()) {
if (signal_pending(current)) {
if (ret == 0)
ret = -ERESTARTSYS;
break;
}
schedule();
}
extract_crng(tmp);
i = min_t(int, nbytes, CHACHA20_BLOCK_SIZE);
if (copy_to_user(buf, tmp, i)) {
ret = -EFAULT;
break;
}
nbytes -= i;
buf += i;
ret += i;
}
/* Wipe data just written to memory */
memzero_explicit(tmp, sizeof(tmp));
return ret;
}
/********************************************************************* /*********************************************************************
* *
* Entropy input management * Entropy input management
...@@ -750,12 +908,12 @@ struct timer_rand_state { ...@@ -750,12 +908,12 @@ struct timer_rand_state {
#define INIT_TIMER_RAND_STATE { INITIAL_JIFFIES, }; #define INIT_TIMER_RAND_STATE { INITIAL_JIFFIES, };
/* /*
* Add device- or boot-specific data to the input and nonblocking * Add device- or boot-specific data to the input pool to help
* pools to help initialize them to unique values. * initialize it.
* *
* None of this adds any entropy, it is meant to avoid the * None of this adds any entropy; it is meant to avoid the problem of
* problem of the nonblocking pool having similar initial state * the entropy pool having similar initial state across largely
* across largely identical devices. * identical devices.
*/ */
void add_device_randomness(const void *buf, unsigned int size) void add_device_randomness(const void *buf, unsigned int size)
{ {
...@@ -767,11 +925,6 @@ void add_device_randomness(const void *buf, unsigned int size) ...@@ -767,11 +925,6 @@ void add_device_randomness(const void *buf, unsigned int size)
_mix_pool_bytes(&input_pool, buf, size); _mix_pool_bytes(&input_pool, buf, size);
_mix_pool_bytes(&input_pool, &time, sizeof(time)); _mix_pool_bytes(&input_pool, &time, sizeof(time));
spin_unlock_irqrestore(&input_pool.lock, flags); spin_unlock_irqrestore(&input_pool.lock, flags);
spin_lock_irqsave(&nonblocking_pool.lock, flags);
_mix_pool_bytes(&nonblocking_pool, buf, size);
_mix_pool_bytes(&nonblocking_pool, &time, sizeof(time));
spin_unlock_irqrestore(&nonblocking_pool.lock, flags);
} }
EXPORT_SYMBOL(add_device_randomness); EXPORT_SYMBOL(add_device_randomness);
...@@ -802,7 +955,7 @@ static void add_timer_randomness(struct timer_rand_state *state, unsigned num) ...@@ -802,7 +955,7 @@ static void add_timer_randomness(struct timer_rand_state *state, unsigned num)
sample.jiffies = jiffies; sample.jiffies = jiffies;
sample.cycles = random_get_entropy(); sample.cycles = random_get_entropy();
sample.num = num; sample.num = num;
r = nonblocking_pool.initialized ? &input_pool : &nonblocking_pool; r = &input_pool;
mix_pool_bytes(r, &sample, sizeof(sample)); mix_pool_bytes(r, &sample, sizeof(sample));
/* /*
...@@ -918,11 +1071,21 @@ void add_interrupt_randomness(int irq, int irq_flags) ...@@ -918,11 +1071,21 @@ void add_interrupt_randomness(int irq, int irq_flags)
fast_mix(fast_pool); fast_mix(fast_pool);
add_interrupt_bench(cycles); add_interrupt_bench(cycles);
if (!crng_ready()) {
if ((fast_pool->count >= 64) &&
crng_fast_load((char *) fast_pool->pool,
sizeof(fast_pool->pool))) {
fast_pool->count = 0;
fast_pool->last = now;
}
return;
}
if ((fast_pool->count < 64) && if ((fast_pool->count < 64) &&
!time_after(now, fast_pool->last + HZ)) !time_after(now, fast_pool->last + HZ))
return; return;
r = nonblocking_pool.initialized ? &input_pool : &nonblocking_pool; r = &input_pool;
if (!spin_trylock(&r->lock)) if (!spin_trylock(&r->lock))
return; return;
...@@ -966,9 +1129,6 @@ EXPORT_SYMBOL_GPL(add_disk_randomness); ...@@ -966,9 +1129,6 @@ EXPORT_SYMBOL_GPL(add_disk_randomness);
* *
*********************************************************************/ *********************************************************************/
static ssize_t extract_entropy(struct entropy_store *r, void *buf,
size_t nbytes, int min, int rsvd);
/* /*
* This utility inline function is responsible for transferring entropy * This utility inline function is responsible for transferring entropy
* from the primary pool to the secondary extraction pool. We make * from the primary pool to the secondary extraction pool. We make
...@@ -1143,6 +1303,36 @@ static void extract_buf(struct entropy_store *r, __u8 *out) ...@@ -1143,6 +1303,36 @@ static void extract_buf(struct entropy_store *r, __u8 *out)
memzero_explicit(&hash, sizeof(hash)); memzero_explicit(&hash, sizeof(hash));
} }
static ssize_t _extract_entropy(struct entropy_store *r, void *buf,
size_t nbytes, int fips)
{
ssize_t ret = 0, i;
__u8 tmp[EXTRACT_SIZE];
unsigned long flags;
while (nbytes) {
extract_buf(r, tmp);
if (fips) {
spin_lock_irqsave(&r->lock, flags);
if (!memcmp(tmp, r->last_data, EXTRACT_SIZE))
panic("Hardware RNG duplicated output!\n");
memcpy(r->last_data, tmp, EXTRACT_SIZE);
spin_unlock_irqrestore(&r->lock, flags);
}
i = min_t(int, nbytes, EXTRACT_SIZE);
memcpy(buf, tmp, i);
nbytes -= i;
buf += i;
ret += i;
}
/* Wipe data just returned from memory */
memzero_explicit(tmp, sizeof(tmp));
return ret;
}
/* /*
* This function extracts randomness from the "entropy pool", and * This function extracts randomness from the "entropy pool", and
* returns it in a buffer. * returns it in a buffer.
...@@ -1155,7 +1345,6 @@ static void extract_buf(struct entropy_store *r, __u8 *out) ...@@ -1155,7 +1345,6 @@ static void extract_buf(struct entropy_store *r, __u8 *out)
static ssize_t extract_entropy(struct entropy_store *r, void *buf, static ssize_t extract_entropy(struct entropy_store *r, void *buf,
size_t nbytes, int min, int reserved) size_t nbytes, int min, int reserved)
{ {
ssize_t ret = 0, i;
__u8 tmp[EXTRACT_SIZE]; __u8 tmp[EXTRACT_SIZE];
unsigned long flags; unsigned long flags;
...@@ -1179,27 +1368,7 @@ static ssize_t extract_entropy(struct entropy_store *r, void *buf, ...@@ -1179,27 +1368,7 @@ static ssize_t extract_entropy(struct entropy_store *r, void *buf,
xfer_secondary_pool(r, nbytes); xfer_secondary_pool(r, nbytes);
nbytes = account(r, nbytes, min, reserved); nbytes = account(r, nbytes, min, reserved);
while (nbytes) { return _extract_entropy(r, buf, nbytes, fips_enabled);
extract_buf(r, tmp);
if (fips_enabled) {
spin_lock_irqsave(&r->lock, flags);
if (!memcmp(tmp, r->last_data, EXTRACT_SIZE))
panic("Hardware RNG duplicated output!\n");
memcpy(r->last_data, tmp, EXTRACT_SIZE);
spin_unlock_irqrestore(&r->lock, flags);
}
i = min_t(int, nbytes, EXTRACT_SIZE);
memcpy(buf, tmp, i);
nbytes -= i;
buf += i;
ret += i;
}
/* Wipe data just returned from memory */
memzero_explicit(tmp, sizeof(tmp));
return ret;
} }
/* /*
...@@ -1254,15 +1423,26 @@ static ssize_t extract_entropy_user(struct entropy_store *r, void __user *buf, ...@@ -1254,15 +1423,26 @@ static ssize_t extract_entropy_user(struct entropy_store *r, void __user *buf,
*/ */
void get_random_bytes(void *buf, int nbytes) void get_random_bytes(void *buf, int nbytes)
{ {
__u8 tmp[CHACHA20_BLOCK_SIZE];
#if DEBUG_RANDOM_BOOT > 0 #if DEBUG_RANDOM_BOOT > 0
if (unlikely(nonblocking_pool.initialized == 0)) if (!crng_ready())
printk(KERN_NOTICE "random: %pF get_random_bytes called " printk(KERN_NOTICE "random: %pF get_random_bytes called "
"with %d bits of entropy available\n", "with crng_init = %d\n", (void *) _RET_IP_, crng_init);
(void *) _RET_IP_,
nonblocking_pool.entropy_total);
#endif #endif
trace_get_random_bytes(nbytes, _RET_IP_); trace_get_random_bytes(nbytes, _RET_IP_);
extract_entropy(&nonblocking_pool, buf, nbytes, 0,