vmstat.h 7.98 KB
Newer Older
1 2 3 4 5
#ifndef _LINUX_VMSTAT_H
#define _LINUX_VMSTAT_H

#include <linux/types.h>
#include <linux/percpu.h>
Christoph Lameter's avatar
Christoph Lameter committed
6
#include <linux/mm.h>
7 8
#include <linux/mmzone.h>
#include <asm/atomic.h>
9

10 11 12 13 14 15
#ifdef CONFIG_ZONE_DMA
#define DMA_ZONE(xx) xx##_DMA,
#else
#define DMA_ZONE(xx)
#endif

16 17 18 19 20 21 22 23 24 25 26 27
#ifdef CONFIG_ZONE_DMA32
#define DMA32_ZONE(xx) xx##_DMA32,
#else
#define DMA32_ZONE(xx)
#endif

#ifdef CONFIG_HIGHMEM
#define HIGHMEM_ZONE(xx) , xx##_HIGH
#else
#define HIGHMEM_ZONE(xx)
#endif

28

Mel Gorman's avatar
Mel Gorman committed
29
#define FOR_ALL_ZONES(xx) DMA_ZONE(xx) DMA32_ZONE(xx) xx##_NORMAL HIGHMEM_ZONE(xx) , xx##_MOVABLE
30 31 32 33 34 35 36 37 38

enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
		FOR_ALL_ZONES(PGALLOC),
		PGFREE, PGACTIVATE, PGDEACTIVATE,
		PGFAULT, PGMAJFAULT,
		FOR_ALL_ZONES(PGREFILL),
		FOR_ALL_ZONES(PGSTEAL),
		FOR_ALL_ZONES(PGSCAN_KSWAPD),
		FOR_ALL_ZONES(PGSCAN_DIRECT),
39 40 41
#ifdef CONFIG_NUMA
		PGSCAN_ZONE_RECLAIM_FAILED,
#endif
42
		PGINODESTEAL, SLABS_SCANNED, KSWAPD_STEAL, KSWAPD_INODESTEAL,
43 44
		KSWAPD_LOW_WMARK_HIT_QUICKLY, KSWAPD_HIGH_WMARK_HIT_QUICKLY,
		KSWAPD_SKIP_CONGESTION_WAIT,
45
		PAGEOUTRUN, ALLOCSTALL, PGROTATED,
46 47
#ifdef CONFIG_COMPACTION
		COMPACTBLOCKS, COMPACTPAGES, COMPACTPAGEFAILED,
48
		COMPACTSTALL, COMPACTFAIL, COMPACTSUCCESS,
49
#endif
50 51
#ifdef CONFIG_HUGETLB_PAGE
		HTLB_BUDDY_PGALLOC, HTLB_BUDDY_PGALLOC_FAIL,
52 53 54 55
#endif
		UNEVICTABLE_PGCULLED,	/* culled to noreclaim list */
		UNEVICTABLE_PGSCANNED,	/* scanned for reclaimability */
		UNEVICTABLE_PGRESCUED,	/* rescued from noreclaim list */
Nick Piggin's avatar
Nick Piggin committed
56 57 58 59
		UNEVICTABLE_PGMLOCKED,
		UNEVICTABLE_PGMUNLOCKED,
		UNEVICTABLE_PGCLEARED,	/* on COW, page truncate */
		UNEVICTABLE_PGSTRANDED,	/* unable to isolate on unlock */
60
		UNEVICTABLE_MLOCKFREED,
61 62 63
		NR_VM_EVENT_ITEMS
};

Adrian Bunk's avatar
Adrian Bunk committed
64 65
extern int sysctl_stat_interval;

66 67 68 69 70 71 72 73 74 75 76
#ifdef CONFIG_VM_EVENT_COUNTERS
/*
 * Light weight per cpu counter implementation.
 *
 * Counters should only be incremented and no critical kernel component
 * should rely on the counter values.
 *
 * Counters are handled completely inline. On many platforms the code
 * generated will simply be the increment of a global address.
 */

77 78
struct vm_event_state {
	unsigned long event[NR_VM_EVENT_ITEMS];
79 80
};

81 82 83 84
DECLARE_PER_CPU(struct vm_event_state, vm_event_states);

static inline void __count_vm_event(enum vm_event_item item)
{
85
	__this_cpu_inc(vm_event_states.event[item]);
86 87 88 89
}

static inline void count_vm_event(enum vm_event_item item)
{
90
	this_cpu_inc(vm_event_states.event[item]);
91 92 93 94
}

static inline void __count_vm_events(enum vm_event_item item, long delta)
{
95
	__this_cpu_add(vm_event_states.event[item], delta);
96 97 98 99
}

static inline void count_vm_events(enum vm_event_item item, long delta)
{
100
	this_cpu_add(vm_event_states.event[item], delta);
101 102 103
}

extern void all_vm_events(unsigned long *);
104
#ifdef CONFIG_HOTPLUG
105
extern void vm_events_fold_cpu(int cpu);
106 107 108 109 110
#else
static inline void vm_events_fold_cpu(int cpu)
{
}
#endif
111 112 113 114

#else

/* Disable counters */
115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
static inline void count_vm_event(enum vm_event_item item)
{
}
static inline void count_vm_events(enum vm_event_item item, long delta)
{
}
static inline void __count_vm_event(enum vm_event_item item)
{
}
static inline void __count_vm_events(enum vm_event_item item, long delta)
{
}
static inline void all_vm_events(unsigned long *ret)
{
}
static inline void vm_events_fold_cpu(int cpu)
{
}
133 134 135 136

#endif /* CONFIG_VM_EVENT_COUNTERS */

#define __count_zone_vm_events(item, zone, delta) \
137 138
		__count_vm_events(item##_NORMAL - ZONE_NORMAL + \
		zone_idx(zone), delta)
139

140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
/*
 * Zone based page accounting with per cpu differentials.
 */
extern atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];

static inline void zone_page_state_add(long x, struct zone *zone,
				 enum zone_stat_item item)
{
	atomic_long_add(x, &zone->vm_stat[item]);
	atomic_long_add(x, &vm_stat[item]);
}

static inline unsigned long global_page_state(enum zone_stat_item item)
{
	long x = atomic_long_read(&vm_stat[item]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
}

static inline unsigned long zone_page_state(struct zone *zone,
					enum zone_stat_item item)
{
	long x = atomic_long_read(&zone->vm_stat[item]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
}

173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194
/*
 * More accurate version that also considers the currently pending
 * deltas. For that we need to loop over all cpus to find the current
 * deltas. There is no synchronization so the result cannot be
 * exactly accurate either.
 */
static inline unsigned long zone_page_state_snapshot(struct zone *zone,
					enum zone_stat_item item)
{
	long x = atomic_long_read(&zone->vm_stat[item]);

#ifdef CONFIG_SMP
	int cpu;
	for_each_online_cpu(cpu)
		x += per_cpu_ptr(zone->pageset, cpu)->vm_stat_diff[item];

	if (x < 0)
		x = 0;
#endif
	return x;
}

195 196
extern unsigned long global_reclaimable_pages(void);
extern unsigned long zone_reclaimable_pages(struct zone *zone);
197

198 199 200 201 202 203 204 205 206 207 208 209
#ifdef CONFIG_NUMA
/*
 * Determine the per node value of a stat item. This function
 * is called frequently in a NUMA machine, so try to be as
 * frugal as possible.
 */
static inline unsigned long node_page_state(int node,
				 enum zone_stat_item item)
{
	struct zone *zones = NODE_DATA(node)->node_zones;

	return
210 211 212
#ifdef CONFIG_ZONE_DMA
		zone_page_state(&zones[ZONE_DMA], item) +
#endif
213
#ifdef CONFIG_ZONE_DMA32
214 215 216 217 218
		zone_page_state(&zones[ZONE_DMA32], item) +
#endif
#ifdef CONFIG_HIGHMEM
		zone_page_state(&zones[ZONE_HIGHMEM], item) +
#endif
Mel Gorman's avatar
Mel Gorman committed
219 220
		zone_page_state(&zones[ZONE_NORMAL], item) +
		zone_page_state(&zones[ZONE_MOVABLE], item);
221
}
222

Andi Kleen's avatar
Andi Kleen committed
223
extern void zone_statistics(struct zone *, struct zone *, gfp_t gfp);
224

225
#else
226

227
#define node_page_state(node, item) global_page_state(item)
Andi Kleen's avatar
Andi Kleen committed
228
#define zone_statistics(_zl, _z, gfp) do { } while (0)
229 230

#endif /* CONFIG_NUMA */
231 232 233 234 235 236 237 238 239

#define add_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, __d)
#define sub_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, -(__d))

static inline void zap_zone_vm_stats(struct zone *zone)
{
	memset(zone->vm_stat, 0, sizeof(zone->vm_stat));
}

240 241
extern void inc_zone_state(struct zone *, enum zone_stat_item);

242 243 244 245
#ifdef CONFIG_SMP
void __mod_zone_page_state(struct zone *, enum zone_stat_item item, int);
void __inc_zone_page_state(struct page *, enum zone_stat_item);
void __dec_zone_page_state(struct page *, enum zone_stat_item);
246

247 248 249 250 251
void mod_zone_page_state(struct zone *, enum zone_stat_item, int);
void inc_zone_page_state(struct page *, enum zone_stat_item);
void dec_zone_page_state(struct page *, enum zone_stat_item);

extern void inc_zone_state(struct zone *, enum zone_stat_item);
252 253 254
extern void __inc_zone_state(struct zone *, enum zone_stat_item);
extern void dec_zone_state(struct zone *, enum zone_stat_item);
extern void __dec_zone_state(struct zone *, enum zone_stat_item);
255 256

void refresh_cpu_vm_stats(int);
257 258 259 260 261

int calculate_pressure_threshold(struct zone *zone);
int calculate_normal_threshold(struct zone *zone);
void set_pgdat_percpu_threshold(pg_data_t *pgdat,
				int (*calculate_pressure)(struct zone *));
262 263 264 265 266 267 268 269 270 271 272 273
#else /* CONFIG_SMP */

/*
 * We do not maintain differentials in a single processor configuration.
 * The functions directly modify the zone and global counters.
 */
static inline void __mod_zone_page_state(struct zone *zone,
			enum zone_stat_item item, int delta)
{
	zone_page_state_add(delta, zone, item);
}

274 275 276 277 278 279
static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
{
	atomic_long_inc(&zone->vm_stat[item]);
	atomic_long_inc(&vm_stat[item]);
}

280 281 282
static inline void __inc_zone_page_state(struct page *page,
			enum zone_stat_item item)
{
283
	__inc_zone_state(page_zone(page), item);
284 285
}

286 287 288 289 290 291
static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
{
	atomic_long_dec(&zone->vm_stat[item]);
	atomic_long_dec(&vm_stat[item]);
}

292 293 294
static inline void __dec_zone_page_state(struct page *page,
			enum zone_stat_item item)
{
295
	__dec_zone_state(page_zone(page), item);
296 297 298 299 300 301 302 303 304 305
}

/*
 * We only use atomic operations to update counters. So there is no need to
 * disable interrupts.
 */
#define inc_zone_page_state __inc_zone_page_state
#define dec_zone_page_state __dec_zone_page_state
#define mod_zone_page_state __mod_zone_page_state

306
#define set_pgdat_percpu_threshold(pgdat, callback) { }
307

308 309 310 311
static inline void refresh_cpu_vm_stats(int cpu) { }
#endif

#endif /* _LINUX_VMSTAT_H */