snapshot.c 10.8 KB
Newer Older
1
/*
2
 * linux/kernel/power/snapshot.c
3
 *
4
 * This file provide system snapshot/restore functionality.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
 *
 * Copyright (C) 1998-2005 Pavel Machek <pavel@suse.cz>
 *
 * This file is released under the GPLv2, and is based on swsusp.c.
 *
 */


#include <linux/module.h>
#include <linux/mm.h>
#include <linux/suspend.h>
#include <linux/smp_lock.h>
#include <linux/delay.h>
#include <linux/bitops.h>
#include <linux/spinlock.h>
#include <linux/kernel.h>
#include <linux/pm.h>
#include <linux/device.h>
#include <linux/bootmem.h>
#include <linux/syscalls.h>
#include <linux/console.h>
#include <linux/highmem.h>

#include <asm/uaccess.h>
#include <asm/mmu_context.h>
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <asm/io.h>

#include "power.h"

#ifdef CONFIG_HIGHMEM
struct highmem_page {
	char *data;
	struct page *page;
	struct highmem_page *next;
};

static struct highmem_page *highmem_copy;

static int save_highmem_zone(struct zone *zone)
{
	unsigned long zone_pfn;
	mark_free_pages(zone);
	for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) {
		struct page *page;
		struct highmem_page *save;
		void *kaddr;
		unsigned long pfn = zone_pfn + zone->zone_start_pfn;

		if (!(pfn%1000))
			printk(".");
		if (!pfn_valid(pfn))
			continue;
		page = pfn_to_page(pfn);
		/*
		 * This condition results from rvmalloc() sans vmalloc_32()
		 * and architectural memory reservations. This should be
		 * corrected eventually when the cases giving rise to this
		 * are better understood.
		 */
		if (PageReserved(page)) {
			printk("highmem reserved page?!\n");
			continue;
		}
		BUG_ON(PageNosave(page));
		if (PageNosaveFree(page))
			continue;
		save = kmalloc(sizeof(struct highmem_page), GFP_ATOMIC);
		if (!save)
			return -ENOMEM;
		save->next = highmem_copy;
		save->page = page;
		save->data = (void *) get_zeroed_page(GFP_ATOMIC);
		if (!save->data) {
			kfree(save);
			return -ENOMEM;
		}
		kaddr = kmap_atomic(page, KM_USER0);
		memcpy(save->data, kaddr, PAGE_SIZE);
		kunmap_atomic(kaddr, KM_USER0);
		highmem_copy = save;
	}
	return 0;
}


static int save_highmem(void)
{
	struct zone *zone;
	int res = 0;

	pr_debug("swsusp: Saving Highmem\n");
	for_each_zone (zone) {
		if (is_highmem(zone))
			res = save_highmem_zone(zone);
		if (res)
			return res;
	}
	return 0;
}

int restore_highmem(void)
{
	printk("swsusp: Restoring Highmem\n");
	while (highmem_copy) {
		struct highmem_page *save = highmem_copy;
		void *kaddr;
		highmem_copy = save->next;

		kaddr = kmap_atomic(save->page, KM_USER0);
		memcpy(kaddr, save->data, PAGE_SIZE);
		kunmap_atomic(kaddr, KM_USER0);
		free_page((long) save->data);
		kfree(save);
	}
	return 0;
}
Pavel Machek's avatar
Pavel Machek committed
123
124
125
126
#else
static int save_highmem(void) { return 0; }
int restore_highmem(void) { return 0; }
#endif /* CONFIG_HIGHMEM */
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144


static int pfn_is_nosave(unsigned long pfn)
{
	unsigned long nosave_begin_pfn = __pa(&__nosave_begin) >> PAGE_SHIFT;
	unsigned long nosave_end_pfn = PAGE_ALIGN(__pa(&__nosave_end)) >> PAGE_SHIFT;
	return (pfn >= nosave_begin_pfn) && (pfn < nosave_end_pfn);
}

/**
 *	saveable - Determine whether a page should be cloned or not.
 *	@pfn:	The page
 *
 *	We save a page if it's Reserved, and not in the range of pages
 *	statically defined as 'unsaveable', or if it isn't reserved, and
 *	isn't part of a free chunk of pages.
 */

Pavel Machek's avatar
Pavel Machek committed
145
static int saveable(struct zone *zone, unsigned long *zone_pfn)
146
147
{
	unsigned long pfn = *zone_pfn + zone->zone_start_pfn;
Pavel Machek's avatar
Pavel Machek committed
148
	struct page *page;
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166

	if (!pfn_valid(pfn))
		return 0;

	page = pfn_to_page(pfn);
	BUG_ON(PageReserved(page) && PageNosave(page));
	if (PageNosave(page))
		return 0;
	if (PageReserved(page) && pfn_is_nosave(pfn)) {
		pr_debug("[nosave pfn 0x%lx]", pfn);
		return 0;
	}
	if (PageNosaveFree(page))
		return 0;

	return 1;
}

167
static unsigned count_data_pages(void)
168
169
170
{
	struct zone *zone;
	unsigned long zone_pfn;
Pavel Machek's avatar
Pavel Machek committed
171
	unsigned int n = 0;
172
173
174
175
176
177

	for_each_zone (zone) {
		if (is_highmem(zone))
			continue;
		mark_free_pages(zone);
		for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn)
178
			n += saveable(zone, &zone_pfn);
179
	}
180
	return n;
181
182
}

183
static void copy_data_pages(struct pbe *pblist)
184
185
186
{
	struct zone *zone;
	unsigned long zone_pfn;
187
	struct pbe *pbe, *p;
188

189
	pbe = pblist;
190
191
192
193
194
	for_each_zone (zone) {
		if (is_highmem(zone))
			continue;
		mark_free_pages(zone);
		/* This is necessary for swsusp_free() */
195
		for_each_pb_page (p, pblist)
196
			SetPageNosaveFree(virt_to_page(p));
197
		for_each_pbe (p, pblist)
198
199
200
			SetPageNosaveFree(virt_to_page(p->address));
		for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) {
			if (saveable(zone, &zone_pfn)) {
Pavel Machek's avatar
Pavel Machek committed
201
				struct page *page;
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
				page = pfn_to_page(zone_pfn + zone->zone_start_pfn);
				BUG_ON(!pbe);
				pbe->orig_address = (unsigned long)page_address(page);
				/* copy_page is not usable for copying task structs. */
				memcpy((void *)pbe->address, (void *)pbe->orig_address, PAGE_SIZE);
				pbe = pbe->next;
			}
		}
	}
	BUG_ON(pbe);
}


/**
 *	free_pagedir - free pages allocated with alloc_pagedir()
 */

219
void free_pagedir(struct pbe *pblist)
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
{
	struct pbe *pbe;

	while (pblist) {
		pbe = (pblist + PB_PAGE_SKIP)->next;
		ClearPageNosave(virt_to_page(pblist));
		ClearPageNosaveFree(virt_to_page(pblist));
		free_page((unsigned long)pblist);
		pblist = pbe;
	}
}

/**
 *	fill_pb_page - Create a list of PBEs on a given memory page
 */

static inline void fill_pb_page(struct pbe *pbpage)
{
	struct pbe *p;

	p = pbpage;
	pbpage += PB_PAGE_SKIP;
	do
		p->next = p + 1;
	while (++p < pbpage);
}

/**
 *	create_pbe_list - Create a list of PBEs on top of a given chain
 *	of memory pages allocated with alloc_pagedir()
 */

Pavel Machek's avatar
Pavel Machek committed
252
void create_pbe_list(struct pbe *pblist, unsigned int nr_pages)
253
254
{
	struct pbe *pbpage, *p;
Pavel Machek's avatar
Pavel Machek committed
255
	unsigned int num = PBES_PER_PAGE;
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271

	for_each_pb_page (pbpage, pblist) {
		if (num >= nr_pages)
			break;

		fill_pb_page(pbpage);
		num += PBES_PER_PAGE;
	}
	if (pbpage) {
		for (num -= PBES_PER_PAGE - 1, p = pbpage; num < nr_pages; p++, num++)
			p->next = p + 1;
		p->next = NULL;
	}
	pr_debug("create_pbe_list(): initialized %d PBEs\n", num);
}

272
273
274
275
276
277
278
279
280
281
282
283
/**
 *	@safe_needed - on resume, for storing the PBE list and the image,
 *	we can only use memory pages that do not conflict with the pages
 *	which had been used before suspend.
 *
 *	The unsafe pages are marked with the PG_nosave_free flag
 *
 *	Allocated but unusable (ie eaten) memory pages should be marked
 *	so that swsusp_free() can release them
 */

static inline void *alloc_image_page(gfp_t gfp_mask, int safe_needed)
284
{
285
286
287
288
289
290
291
292
293
294
295
	void *res;

	if (safe_needed)
		do {
			res = (void *)get_zeroed_page(gfp_mask);
			if (res && PageNosaveFree(virt_to_page(res)))
				/* This is for swsusp_free() */
				SetPageNosave(virt_to_page(res));
		} while (res && PageNosaveFree(virt_to_page(res)));
	else
		res = (void *)get_zeroed_page(gfp_mask);
296
297
298
299
300
301
302
	if (res) {
		SetPageNosave(virt_to_page(res));
		SetPageNosaveFree(virt_to_page(res));
	}
	return res;
}

303
304
305
306
307
unsigned long get_safe_page(gfp_t gfp_mask)
{
	return (unsigned long)alloc_image_page(gfp_mask, 1);
}

308
309
310
311
312
313
314
315
316
317
318
319
320
/**
 *	alloc_pagedir - Allocate the page directory.
 *
 *	First, determine exactly how many pages we need and
 *	allocate them.
 *
 *	We arrange the pages in a chain: each page is an array of PBES_PER_PAGE
 *	struct pbe elements (pbes) and the last element in the page points
 *	to the next page.
 *
 *	On each page we set up a list of struct_pbe elements.
 */

321
struct pbe *alloc_pagedir(unsigned int nr_pages, gfp_t gfp_mask, int safe_needed)
322
{
Pavel Machek's avatar
Pavel Machek committed
323
	unsigned int num;
324
325
326
327
328
329
	struct pbe *pblist, *pbe;

	if (!nr_pages)
		return NULL;

	pr_debug("alloc_pagedir(): nr_pages = %d\n", nr_pages);
330
	pblist = alloc_image_page(gfp_mask, safe_needed);
331
332
333
334
	/* FIXME: rewrite this ugly loop */
	for (pbe = pblist, num = PBES_PER_PAGE; pbe && num < nr_pages;
        		pbe = pbe->next, num += PBES_PER_PAGE) {
		pbe += PB_PAGE_SKIP;
335
		pbe->next = alloc_image_page(gfp_mask, safe_needed);
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
	}
	if (!pbe) { /* get_zeroed_page() failed */
		free_pagedir(pblist);
		pblist = NULL;
        }
	return pblist;
}

/**
 * Free pages we allocated for suspend. Suspend pages are alocated
 * before atomic copy, so we need to free them after resume.
 */

void swsusp_free(void)
{
	struct zone *zone;
	unsigned long zone_pfn;

	for_each_zone(zone) {
		for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn)
			if (pfn_valid(zone_pfn + zone->zone_start_pfn)) {
Pavel Machek's avatar
Pavel Machek committed
357
				struct page *page;
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
				page = pfn_to_page(zone_pfn + zone->zone_start_pfn);
				if (PageNosave(page) && PageNosaveFree(page)) {
					ClearPageNosave(page);
					ClearPageNosaveFree(page);
					free_page((long) page_address(page));
				}
			}
	}
}


/**
 *	enough_free_mem - Make sure we enough free memory to snapshot.
 *
 *	Returns TRUE or FALSE after checking the number of available
 *	free pages.
 */

Pavel Machek's avatar
Pavel Machek committed
376
static int enough_free_mem(unsigned int nr_pages)
377
378
{
	pr_debug("swsusp: available memory: %u pages\n", nr_free_pages());
379
380
	return nr_free_pages() > (nr_pages + PAGES_FOR_IO +
		(nr_pages + PBES_PER_PAGE - 1) / PBES_PER_PAGE);
381
382
}

383
384
385
386
387
388
389
390
391
392
393
int alloc_data_pages(struct pbe *pblist, gfp_t gfp_mask, int safe_needed)
{
	struct pbe *p;

	for_each_pbe (p, pblist) {
		p->address = (unsigned long)alloc_image_page(gfp_mask, safe_needed);
		if (!p->address)
			return -ENOMEM;
	}
	return 0;
}
394

Pavel Machek's avatar
Pavel Machek committed
395
static struct pbe *swsusp_alloc(unsigned int nr_pages)
396
{
397
	struct pbe *pblist;
398

399
	if (!(pblist = alloc_pagedir(nr_pages, GFP_ATOMIC | __GFP_COLD, 0))) {
400
		printk(KERN_ERR "suspend: Allocating pagedir failed.\n");
401
		return NULL;
402
	}
403
	create_pbe_list(pblist, nr_pages);
404

405
406
407
408
	if (alloc_data_pages(pblist, GFP_ATOMIC | __GFP_COLD, 0)) {
		printk(KERN_ERR "suspend: Allocating image pages failed.\n");
		swsusp_free();
		return NULL;
409
410
	}

411
	return pblist;
412
413
}

414
asmlinkage int swsusp_save(void)
415
{
Pavel Machek's avatar
Pavel Machek committed
416
	unsigned int nr_pages;
417
418
419
420
421
422
423
424
425

	pr_debug("swsusp: critical section: \n");
	if (save_highmem()) {
		printk(KERN_CRIT "swsusp: Not enough free pages for highmem\n");
		restore_highmem();
		return -ENOMEM;
	}

	drain_local_pages();
426
427
	nr_pages = count_data_pages();
	printk("swsusp: Need to copy %u pages\n", nr_pages);
428
429

	pr_debug("swsusp: pages needed: %u + %lu + %u, free: %u\n",
430
431
		 nr_pages,
		 (nr_pages + PBES_PER_PAGE - 1) / PBES_PER_PAGE,
432
433
		 PAGES_FOR_IO, nr_free_pages());

434
435
436
437
438
	/* This is needed because of the fixed size of swsusp_info */
	if (MAX_PBES < (nr_pages + PBES_PER_PAGE - 1) / PBES_PER_PAGE)
		return -ENOSPC;

	if (!enough_free_mem(nr_pages)) {
439
440
441
442
		printk(KERN_ERR "swsusp: Not enough free memory\n");
		return -ENOMEM;
	}

443
	if (!enough_swap(nr_pages)) {
444
445
446
447
		printk(KERN_ERR "swsusp: Not enough free swap\n");
		return -ENOSPC;
	}

448
449
450
	pagedir_nosave = swsusp_alloc(nr_pages);
	if (!pagedir_nosave)
		return -ENOMEM;
451
452
453
454
455

	/* During allocating of suspend pagedir, new cold pages may appear.
	 * Kill them.
	 */
	drain_local_pages();
456
	copy_data_pages(pagedir_nosave);
457
458
459
460
461
462
463

	/*
	 * End of critical section. From now on, we can write to memory,
	 * but we should not touch disk. This specially means we must _not_
	 * touch swap space! Except we must write out our image of course.
	 */

464
465
466
	nr_copy_pages = nr_pages;

	printk("swsusp: critical section/: done (%d pages copied)\n", nr_pages);
467
468
	return 0;
}