Commit 1feab10d authored by Jozsef Kadlecsik's avatar Jozsef Kadlecsik Committed by Pablo Neira Ayuso
Browse files

netfilter: ipset: Unified hash type generation



Signed-off-by: default avatarJozsef Kadlecsik <kadlec@blackhole.kfki.hu>
Signed-off-by: default avatarPablo Neira Ayuso <pablo@netfilter.org>
parent b0da3905
#ifndef _IP_SET_AHASH_H
#define _IP_SET_AHASH_H
/* Copyright (C) 2013 Jozsef Kadlecsik <kadlec@blackhole.kfki.hu>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#ifndef _IP_SET_HASH_GEN_H
#define _IP_SET_HASH_GEN_H
#include <linux/rcupdate.h>
#include <linux/jhash.h>
#include <linux/netfilter/ipset/ip_set_timeout.h>
#ifndef rcu_dereference_bh
#define rcu_dereference_bh(p) rcu_dereference(p)
#endif
#define CONCAT(a, b, c) a##b##c
#define TOKEN(a, b, c) CONCAT(a, b, c)
#define type_pf_next TOKEN(TYPE, PF, _elem)
#define CONCAT(a, b) a##b
#define TOKEN(a, b) CONCAT(a, b)
/* Hashing which uses arrays to resolve clashing. The hash table is resized
* (doubled) when searching becomes too long.
......@@ -71,34 +79,12 @@ struct htable {
#define hbucket(h, i) (&((h)->bucket[i]))
/* Book-keeping of the prefixes added to the set */
struct ip_set_hash_nets {
struct net_prefixes {
u8 cidr; /* the different cidr values in the set */
u32 nets; /* number of elements per cidr */
};
/* The generic ip_set hash structure */
struct ip_set_hash {
struct htable *table; /* the hash table */
u32 maxelem; /* max elements in the hash */
u32 elements; /* current element (vs timeout) */
u32 initval; /* random jhash init value */
u32 timeout; /* timeout value, if enabled */
struct timer_list gc; /* garbage collection when timeout enabled */
struct type_pf_next next; /* temporary storage for uadd */
#ifdef IP_SET_HASH_WITH_MULTI
u8 ahash_max; /* max elements in an array block */
#endif
#ifdef IP_SET_HASH_WITH_NETMASK
u8 netmask; /* netmask value for subnets to store */
#endif
#ifdef IP_SET_HASH_WITH_RBTREE
struct rb_root rbtree;
#endif
#ifdef IP_SET_HASH_WITH_NETS
struct ip_set_hash_nets nets[0]; /* book-keeping of prefixes */
#endif
};
/* Compute the hash table size */
static size_t
htable_size(u8 hbits)
{
......@@ -128,25 +114,188 @@ htable_bits(u32 hashsize)
return bits;
}
/* Destroy the hashtable part of the set */
static void
ahash_destroy(struct htable *t)
{
struct hbucket *n;
u32 i;
for (i = 0; i < jhash_size(t->htable_bits); i++) {
n = hbucket(t, i);
if (n->size)
/* FIXME: use slab cache */
kfree(n->value);
}
ip_set_free(t);
}
static int
hbucket_elem_add(struct hbucket *n, u8 ahash_max, size_t dsize)
{
if (n->pos >= n->size) {
void *tmp;
if (n->size >= ahash_max)
/* Trigger rehashing */
return -EAGAIN;
tmp = kzalloc((n->size + AHASH_INIT_SIZE) * dsize,
GFP_ATOMIC);
if (!tmp)
return -ENOMEM;
if (n->size) {
memcpy(tmp, n->value, n->size * dsize);
kfree(n->value);
}
n->value = tmp;
n->size += AHASH_INIT_SIZE;
}
return 0;
}
#ifdef IP_SET_HASH_WITH_NETS
#ifdef IP_SET_HASH_WITH_NETS_PACKED
/* When cidr is packed with nomatch, cidr - 1 is stored in the entry */
#define CIDR(cidr) (cidr + 1)
#define CIDR(cidr) (cidr + 1)
#else
#define CIDR(cidr) (cidr)
#define CIDR(cidr) (cidr)
#endif
#define SET_HOST_MASK(family) (family == AF_INET ? 32 : 128)
#ifdef IP_SET_HASH_WITH_MULTI
#define NETS_LENGTH(family) (SET_HOST_MASK(family) + 1)
#else
#define NETS_LENGTH(family) SET_HOST_MASK(family)
#endif
#else
#define NETS_LENGTH(family) 0
#endif /* IP_SET_HASH_WITH_NETS */
#define ext_timeout(e, h) \
(unsigned long *)(((void *)(e)) + (h)->offset[IPSET_OFFSET_TIMEOUT])
#endif /* _IP_SET_HASH_GEN_H */
/* Family dependent templates */
#undef ahash_data
#undef mtype_data_equal
#undef mtype_do_data_match
#undef mtype_data_set_flags
#undef mtype_data_reset_flags
#undef mtype_data_netmask
#undef mtype_data_list
#undef mtype_data_next
#undef mtype_elem
#undef mtype_add_cidr
#undef mtype_del_cidr
#undef mtype_ahash_memsize
#undef mtype_flush
#undef mtype_destroy
#undef mtype_gc_init
#undef mtype_same_set
#undef mtype_kadt
#undef mtype_uadt
#undef mtype
#undef mtype_add
#undef mtype_del
#undef mtype_test_cidrs
#undef mtype_test
#undef mtype_expire
#undef mtype_resize
#undef mtype_head
#undef mtype_list
#undef mtype_gc
#undef mtype_gc_init
#undef mtype_variant
#undef mtype_data_match
#undef HKEY
#define mtype_data_equal TOKEN(MTYPE, _data_equal)
#ifdef IP_SET_HASH_WITH_NETS
#define mtype_do_data_match TOKEN(MTYPE, _do_data_match)
#else
#define mtype_do_data_match(d) 1
#endif
#define mtype_data_set_flags TOKEN(MTYPE, _data_set_flags)
#define mtype_data_reset_flags TOKEN(MTYPE, _data_reset_flags)
#define mtype_data_netmask TOKEN(MTYPE, _data_netmask)
#define mtype_data_list TOKEN(MTYPE, _data_list)
#define mtype_data_next TOKEN(MTYPE, _data_next)
#define mtype_elem TOKEN(MTYPE, _elem)
#define mtype_add_cidr TOKEN(MTYPE, _add_cidr)
#define mtype_del_cidr TOKEN(MTYPE, _del_cidr)
#define mtype_ahash_memsize TOKEN(MTYPE, _ahash_memsize)
#define mtype_flush TOKEN(MTYPE, _flush)
#define mtype_destroy TOKEN(MTYPE, _destroy)
#define mtype_gc_init TOKEN(MTYPE, _gc_init)
#define mtype_same_set TOKEN(MTYPE, _same_set)
#define mtype_kadt TOKEN(MTYPE, _kadt)
#define mtype_uadt TOKEN(MTYPE, _uadt)
#define mtype MTYPE
#define mtype_elem TOKEN(MTYPE, _elem)
#define mtype_add TOKEN(MTYPE, _add)
#define mtype_del TOKEN(MTYPE, _del)
#define mtype_test_cidrs TOKEN(MTYPE, _test_cidrs)
#define mtype_test TOKEN(MTYPE, _test)
#define mtype_expire TOKEN(MTYPE, _expire)
#define mtype_resize TOKEN(MTYPE, _resize)
#define mtype_head TOKEN(MTYPE, _head)
#define mtype_list TOKEN(MTYPE, _list)
#define mtype_gc TOKEN(MTYPE, _gc)
#define mtype_variant TOKEN(MTYPE, _variant)
#define mtype_data_match TOKEN(MTYPE, _data_match)
#ifndef HKEY_DATALEN
#define HKEY_DATALEN sizeof(struct mtype_elem)
#endif
#define HKEY(data, initval, htable_bits) \
(jhash2((u32 *)(data), HKEY_DATALEN/sizeof(u32), initval) \
& jhash_mask(htable_bits))
#ifndef htype
#define htype HTYPE
/* The generic hash structure */
struct htype {
struct htable *table; /* the hash table */
u32 maxelem; /* max elements in the hash */
u32 elements; /* current element (vs timeout) */
u32 initval; /* random jhash init value */
u32 timeout; /* timeout value, if enabled */
size_t dsize; /* data struct size */
size_t offset[IPSET_OFFSET_MAX]; /* Offsets to extensions */
struct timer_list gc; /* garbage collection when timeout enabled */
struct mtype_elem next; /* temporary storage for uadd */
#ifdef IP_SET_HASH_WITH_MULTI
u8 ahash_max; /* max elements in an array block */
#endif
#ifdef IP_SET_HASH_WITH_NETMASK
u8 netmask; /* netmask value for subnets to store */
#endif
#ifdef IP_SET_HASH_WITH_RBTREE
struct rb_root rbtree;
#endif
#ifdef IP_SET_HASH_WITH_NETS
struct net_prefixes nets[0]; /* book-keeping of prefixes */
#endif
};
#endif
#ifdef IP_SET_HASH_WITH_NETS
/* Network cidr size book keeping when the hash stores different
* sized networks */
static void
add_cidr(struct ip_set_hash *h, u8 cidr, u8 nets_length)
mtype_add_cidr(struct htype *h, u8 cidr, u8 nets_length)
{
int i, j;
......@@ -172,7 +321,7 @@ add_cidr(struct ip_set_hash *h, u8 cidr, u8 nets_length)
}
static void
del_cidr(struct ip_set_hash *h, u8 cidr, u8 nets_length)
mtype_del_cidr(struct htype *h, u8 cidr, u8 nets_length)
{
u8 i, j;
......@@ -188,51 +337,32 @@ del_cidr(struct ip_set_hash *h, u8 cidr, u8 nets_length)
h->nets[j].nets = h->nets[j + 1].nets;
}
}
#else
#define NETS_LENGTH(family) 0
#endif
/* Destroy the hashtable part of the set */
static void
ahash_destroy(struct htable *t)
{
struct hbucket *n;
u32 i;
for (i = 0; i < jhash_size(t->htable_bits); i++) {
n = hbucket(t, i);
if (n->size)
/* FIXME: use slab cache */
kfree(n->value);
}
ip_set_free(t);
}
/* Calculate the actual memory size of the set data */
static size_t
ahash_memsize(const struct ip_set_hash *h, size_t dsize, u8 nets_length)
mtype_ahash_memsize(const struct htype *h, u8 nets_length)
{
u32 i;
struct htable *t = h->table;
size_t memsize = sizeof(*h)
+ sizeof(*t)
#ifdef IP_SET_HASH_WITH_NETS
+ sizeof(struct ip_set_hash_nets) * nets_length
+ sizeof(struct net_prefixes) * nets_length
#endif
+ jhash_size(t->htable_bits) * sizeof(struct hbucket);
for (i = 0; i < jhash_size(t->htable_bits); i++)
memsize += t->bucket[i].size * dsize;
memsize += t->bucket[i].size * h->dsize;
return memsize;
}
/* Flush a hash type of set: destroy all elements */
static void
ip_set_hash_flush(struct ip_set *set)
mtype_flush(struct ip_set *set)
{
struct ip_set_hash *h = set->data;
struct htype *h = set->data;
struct htable *t = h->table;
struct hbucket *n;
u32 i;
......@@ -246,7 +376,7 @@ ip_set_hash_flush(struct ip_set *set)
}
}
#ifdef IP_SET_HASH_WITH_NETS
memset(h->nets, 0, sizeof(struct ip_set_hash_nets)
memset(h->nets, 0, sizeof(struct net_prefixes)
* NETS_LENGTH(set->family));
#endif
h->elements = 0;
......@@ -254,11 +384,11 @@ ip_set_hash_flush(struct ip_set *set)
/* Destroy a hash type of set */
static void
ip_set_hash_destroy(struct ip_set *set)
mtype_destroy(struct ip_set *set)
{
struct ip_set_hash *h = set->data;
struct htype *h = set->data;
if (with_timeout(h->timeout))
if (set->extensions & IPSET_EXT_TIMEOUT)
del_timer_sync(&h->gc);
ahash_destroy(h->table);
......@@ -270,127 +400,127 @@ ip_set_hash_destroy(struct ip_set *set)
set->data = NULL;
}
#endif /* _IP_SET_AHASH_H */
static void
mtype_gc_init(struct ip_set *set, void (*gc)(unsigned long ul_set))
{
struct htype *h = set->data;
#ifndef HKEY_DATALEN
#define HKEY_DATALEN sizeof(struct type_pf_elem)
#endif
init_timer(&h->gc);
h->gc.data = (unsigned long) set;
h->gc.function = gc;
h->gc.expires = jiffies + IPSET_GC_PERIOD(h->timeout) * HZ;
add_timer(&h->gc);
pr_debug("gc initialized, run in every %u\n",
IPSET_GC_PERIOD(h->timeout));
}
#define HKEY(data, initval, htable_bits) \
(jhash2((u32 *)(data), HKEY_DATALEN/sizeof(u32), initval) \
& jhash_mask(htable_bits))
static bool
mtype_same_set(const struct ip_set *a, const struct ip_set *b)
{
const struct htype *x = a->data;
const struct htype *y = b->data;
/* Type/family dependent function prototypes */
#define type_pf_data_equal TOKEN(TYPE, PF, _data_equal)
#define type_pf_data_isnull TOKEN(TYPE, PF, _data_isnull)
#define type_pf_data_copy TOKEN(TYPE, PF, _data_copy)
#define type_pf_data_zero_out TOKEN(TYPE, PF, _data_zero_out)
#define type_pf_data_netmask TOKEN(TYPE, PF, _data_netmask)
#define type_pf_data_list TOKEN(TYPE, PF, _data_list)
#define type_pf_data_tlist TOKEN(TYPE, PF, _data_tlist)
#define type_pf_data_next TOKEN(TYPE, PF, _data_next)
#define type_pf_data_flags TOKEN(TYPE, PF, _data_flags)
#define type_pf_data_reset_flags TOKEN(TYPE, PF, _data_reset_flags)
#ifdef IP_SET_HASH_WITH_NETS
#define type_pf_data_match TOKEN(TYPE, PF, _data_match)
#else
#define type_pf_data_match(d) 1
/* Resizing changes htable_bits, so we ignore it */
return x->maxelem == y->maxelem &&
x->timeout == y->timeout &&
#ifdef IP_SET_HASH_WITH_NETMASK
x->netmask == y->netmask &&
#endif
#define type_pf_elem TOKEN(TYPE, PF, _elem)
#define type_pf_telem TOKEN(TYPE, PF, _telem)
#define type_pf_data_timeout TOKEN(TYPE, PF, _data_timeout)
#define type_pf_data_expired TOKEN(TYPE, PF, _data_expired)
#define type_pf_data_timeout_set TOKEN(TYPE, PF, _data_timeout_set)
#define type_pf_elem_add TOKEN(TYPE, PF, _elem_add)
#define type_pf_add TOKEN(TYPE, PF, _add)
#define type_pf_del TOKEN(TYPE, PF, _del)
#define type_pf_test_cidrs TOKEN(TYPE, PF, _test_cidrs)
#define type_pf_test TOKEN(TYPE, PF, _test)
#define type_pf_elem_tadd TOKEN(TYPE, PF, _elem_tadd)
#define type_pf_del_telem TOKEN(TYPE, PF, _ahash_del_telem)
#define type_pf_expire TOKEN(TYPE, PF, _expire)
#define type_pf_tadd TOKEN(TYPE, PF, _tadd)
#define type_pf_tdel TOKEN(TYPE, PF, _tdel)
#define type_pf_ttest_cidrs TOKEN(TYPE, PF, _ahash_ttest_cidrs)
#define type_pf_ttest TOKEN(TYPE, PF, _ahash_ttest)
#define type_pf_resize TOKEN(TYPE, PF, _resize)
#define type_pf_tresize TOKEN(TYPE, PF, _tresize)
#define type_pf_flush ip_set_hash_flush
#define type_pf_destroy ip_set_hash_destroy
#define type_pf_head TOKEN(TYPE, PF, _head)
#define type_pf_list TOKEN(TYPE, PF, _list)
#define type_pf_tlist TOKEN(TYPE, PF, _tlist)
#define type_pf_same_set TOKEN(TYPE, PF, _same_set)
#define type_pf_kadt TOKEN(TYPE, PF, _kadt)
#define type_pf_uadt TOKEN(TYPE, PF, _uadt)
#define type_pf_gc TOKEN(TYPE, PF, _gc)
#define type_pf_gc_init TOKEN(TYPE, PF, _gc_init)
#define type_pf_variant TOKEN(TYPE, PF, _variant)
#define type_pf_tvariant TOKEN(TYPE, PF, _tvariant)
/* Flavour without timeout */
a->extensions == b->extensions;
}
/* Get the ith element from the array block n */
#define ahash_data(n, i) \
((struct type_pf_elem *)((n)->value) + (i))
#define ahash_data(n, i, dsize) \
((struct mtype_elem *)((n)->value + ((i) * (dsize))))
/* Add an element to the hash table when resizing the set:
* we spare the maintenance of the internal counters. */
static int
type_pf_elem_add(struct hbucket *n, const struct type_pf_elem *value,
u8 ahash_max, u32 cadt_flags)
/* Delete expired elements from the hashtable */
static void
mtype_expire(struct htype *h, u8 nets_length, size_t dsize)
{
struct type_pf_elem *data;
if (n->pos >= n->size) {
void *tmp;
if (n->size >= ahash_max)
/* Trigger rehashing */
return -EAGAIN;
struct htable *t = h->table;
struct hbucket *n;
struct mtype_elem *data;
u32 i;
int j;
tmp = kzalloc((n->size + AHASH_INIT_SIZE)
* sizeof(struct type_pf_elem),
GFP_ATOMIC);
if (!tmp)
return -ENOMEM;
if (n->size) {
memcpy(tmp, n->value,
sizeof(struct type_pf_elem) * n->size);
for (i = 0; i < jhash_size(t->htable_bits); i++) {
n = hbucket(t, i);
for (j = 0; j < n->pos; j++) {
data = ahash_data(n, j, dsize);
if (ip_set_timeout_expired(ext_timeout(data, h))) {
pr_debug("expired %u/%u\n", i, j);
#ifdef IP_SET_HASH_WITH_NETS
mtype_del_cidr(h, CIDR(data->cidr),
nets_length);
#endif
if (j != n->pos - 1)
/* Not last one */
memcpy(data,
ahash_data(n, n->pos - 1, dsize),
dsize);
n->pos--;
h->elements--;
}
}
if (n->pos + AHASH_INIT_SIZE < n->size) {
void *tmp = kzalloc((n->size - AHASH_INIT_SIZE)
* dsize,
GFP_ATOMIC);
if (!tmp)
/* Still try to delete expired elements */
continue;
n->size -= AHASH_INIT_SIZE;
memcpy(tmp, n->value, n->size * dsize);
kfree(n->value);
n->value = tmp;
}
n->value = tmp;
n->size += AHASH_INIT_SIZE;
}
data = ahash_data(n, n->pos++);
type_pf_data_copy(data, value);
#ifdef IP_SET_HASH_WITH_NETS
/* Resizing won't overwrite stored flags */
if (cadt_flags)
type_pf_data_flags(data, cadt_flags);
#endif
return 0;
}
static void
mtype_gc(unsigned long ul_set)
{
struct ip_set *set = (struct ip_set *) ul_set;
struct htype *h = set->data;
pr_debug("called\n");
write_lock_bh(&set->lock);
mtype_expire(h, NETS_LENGTH(set->family), h->dsize);
write_unlock_bh(&set->lock);
h->gc.expires = jiffies + IPSET_GC_PERIOD(h->timeout) * HZ;
add_timer(&h->gc);
}
/* Resize a hash: create a new hash table with doubling the hashsize
* and inserting the elements to it. Repeat until we succeed or
* fail due to memory pressures. */
static int
type_pf_resize(struct ip_set *set, bool retried)
mtype_resize(struct ip_set *set, bool retried)
{
struct ip_set_hash *h = set->data;
struct htype *h = set->data;
struct htable *t, *orig = h->table;
u8 htable_bits = orig->htable_bits;
struct type_pf_elem *data;
#ifdef IP_SET_HASH_WITH_NETS
u8 flags;
#endif
struct mtype_elem *data;
struct mtype_elem *d;
struct hbucket *n, *m;
u32 i, j, flags = 0;
u32 i, j;
int ret;
/* Try to cleanup once */
if (SET_WITH_TIMEOUT(set) && !retried) {
i = h->elements;
write_lock_bh(&set->lock);
mtype_expire(set->data, NETS_LENGTH(set->family),
h->dsize);
write_unlock_bh(&set->lock);
if (h->elements < i)
return 0;
}
retry:
ret = 0;
htable_bits++;
......@@ -412,16 +542,16 @@ type_pf_resize(struct ip_set *set, bool retried)
for (i = 0; i < jhash_size(orig->htable_bits); i++) {
n = hbucket(orig, i);
for (j = 0; j < n->pos; j++) {
data = ahash_data(n, j);
data = ahash_data(n, j, h->dsize);
#ifdef IP_SET_HASH_WITH_NETS
flags = 0;
type_pf_data_reset_flags(data, &flags);
mtype_data_reset_flags(data, &flags);
#endif
m = hbucket(t, HKEY(data, h->initval, htable_bits));
ret = type_pf_elem_add(m, data, AHASH_MAX(h), flags);
ret = hbucket_elem_add(m, AHASH_MAX(h), h->dsize);
if (ret < 0) {
#ifdef IP_SET_HASH_WITH_NETS
type_pf_data_flags(data, flags);
mtype_data_reset_flags(data, &flags);
#endif
read_unlock_bh(&set->lock);
ahash_destroy(t);
......@@ -429,6 +559,11 @@ type_pf_resize(struct ip_set *set, bool retried)
goto retry;
return ret;
}
d = ahash_data(m, m->pos++, h->dsize);
memcpy(d, data, h->dsize);
#ifdef IP_SET_HASH_WITH_NETS
mtype_data_reset_flags(d, &flags);
#endif
}
}
......@@ -445,21 +580,25 @@ type_pf_resize(struct ip_set *set, bool retried)
return 0;
}
static inline void
type_pf_data_next(struct ip_set_hash *h, const struct type_pf_elem *d);
/* Add an element to a hash and update the internal counters when succeeded,
* otherwise report the proper error code. */
static int
type_pf_add(struct ip_set *set, void *value, u32 timeout, u32 flags)
mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
struct ip_set_ext *mext, u32 flags)
{
struct ip_set_hash *h = set->data;
struct htype *h = set->data;
struct htable *t;
const struct type_pf_elem *d = value;
const struct mtype_elem *d = value;
<