dev.c 52.2 KB
Newer Older
1
2
/*
  FUSE: Filesystem in Userspace
Miklos Szeredi's avatar
Miklos Szeredi committed
3
  Copyright (C) 2001-2008  Miklos Szeredi <miklos@szeredi.hu>
4
5
6
7
8
9
10
11
12
13

  This program can be distributed under the terms of the GNU GPL.
  See the file COPYING.
*/

#include "fuse_i.h"

#include <linux/init.h>
#include <linux/module.h>
#include <linux/poll.h>
14
#include <linux/sched/signal.h>
15
16
17
18
19
#include <linux/uio.h>
#include <linux/miscdevice.h>
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/slab.h>
20
#include <linux/pipe_fs_i.h>
21
22
#include <linux/swap.h>
#include <linux/splice.h>
23
#include <linux/sched.h>
24
25

MODULE_ALIAS_MISCDEV(FUSE_MINOR);
26
MODULE_ALIAS("devname:fuse");
27

28
static struct kmem_cache *fuse_req_cachep;
29

30
static struct fuse_dev *fuse_get_dev(struct file *file)
31
{
32
33
34
35
	/*
	 * Lockless access is OK, because file->private data is set
	 * once during mount and is valid until the file is released.
	 */
36
	return ACCESS_ONCE(file->private_data);
37
38
}

39
static void fuse_request_init(struct fuse_req *req, struct page **pages,
40
			      struct fuse_page_desc *page_descs,
41
			      unsigned npages)
42
43
{
	memset(req, 0, sizeof(*req));
44
	memset(pages, 0, sizeof(*pages) * npages);
45
	memset(page_descs, 0, sizeof(*page_descs) * npages);
46
	INIT_LIST_HEAD(&req->list);
47
	INIT_LIST_HEAD(&req->intr_entry);
48
	init_waitqueue_head(&req->waitq);
49
	refcount_set(&req->count, 1);
50
	req->pages = pages;
51
	req->page_descs = page_descs;
52
	req->max_pages = npages;
Miklos Szeredi's avatar
Miklos Szeredi committed
53
	__set_bit(FR_PENDING, &req->flags);
54
55
}

56
static struct fuse_req *__fuse_request_alloc(unsigned npages, gfp_t flags)
57
{
58
59
60
	struct fuse_req *req = kmem_cache_alloc(fuse_req_cachep, flags);
	if (req) {
		struct page **pages;
61
		struct fuse_page_desc *page_descs;
62

63
		if (npages <= FUSE_REQ_INLINE_PAGES) {
64
			pages = req->inline_pages;
65
66
			page_descs = req->inline_page_descs;
		} else {
67
			pages = kmalloc(sizeof(struct page *) * npages, flags);
68
69
70
			page_descs = kmalloc(sizeof(struct fuse_page_desc) *
					     npages, flags);
		}
71

72
73
74
		if (!pages || !page_descs) {
			kfree(pages);
			kfree(page_descs);
75
76
77
78
			kmem_cache_free(fuse_req_cachep, req);
			return NULL;
		}

79
		fuse_request_init(req, pages, page_descs, npages);
80
	}
81
82
	return req;
}
83
84
85
86
87

struct fuse_req *fuse_request_alloc(unsigned npages)
{
	return __fuse_request_alloc(npages, GFP_KERNEL);
}
88
EXPORT_SYMBOL_GPL(fuse_request_alloc);
89

90
struct fuse_req *fuse_request_alloc_nofs(unsigned npages)
Miklos Szeredi's avatar
Miklos Szeredi committed
91
{
92
	return __fuse_request_alloc(npages, GFP_NOFS);
Miklos Szeredi's avatar
Miklos Szeredi committed
93
94
}

95
96
void fuse_request_free(struct fuse_req *req)
{
97
	if (req->pages != req->inline_pages) {
98
		kfree(req->pages);
99
100
		kfree(req->page_descs);
	}
101
102
103
	kmem_cache_free(fuse_req_cachep, req);
}

104
void __fuse_get_request(struct fuse_req *req)
105
{
106
	refcount_inc(&req->count);
107
108
109
110
111
}

/* Must be called with > 1 refcount */
static void __fuse_put_request(struct fuse_req *req)
{
112
	refcount_dec(&req->count);
113
114
}

115
static void fuse_req_init_context(struct fuse_conn *fc, struct fuse_req *req)
116
{
117
118
	req->in.h.uid = from_kuid_munged(&init_user_ns, current_fsuid());
	req->in.h.gid = from_kgid_munged(&init_user_ns, current_fsgid());
119
	req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns);
120
121
}

122
123
124
125
126
127
128
void fuse_set_initialized(struct fuse_conn *fc)
{
	/* Make sure stores before this are seen on another CPU */
	smp_wmb();
	fc->initialized = 1;
}

129
130
131
132
133
static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background)
{
	return !fc->initialized || (for_background && fc->blocked);
}

134
135
136
137
138
139
140
141
142
143
static void fuse_drop_waiting(struct fuse_conn *fc)
{
	if (fc->connected) {
		atomic_dec(&fc->num_waiting);
	} else if (atomic_dec_and_test(&fc->num_waiting)) {
		/* wake up aborters */
		wake_up_all(&fc->blocked_waitq);
	}
}

144
145
static struct fuse_req *__fuse_get_req(struct fuse_conn *fc, unsigned npages,
				       bool for_background)
146
{
147
148
	struct fuse_req *req;
	int err;
149
	atomic_inc(&fc->num_waiting);
150
151
152

	if (fuse_block_alloc(fc, for_background)) {
		err = -EINTR;
153
154
		if (wait_event_killable_exclusive(fc->blocked_waitq,
				!fuse_block_alloc(fc, for_background)))
155
156
			goto out;
	}
157
158
	/* Matches smp_wmb() in fuse_set_initialized() */
	smp_rmb();
159

160
161
162
163
	err = -ENOTCONN;
	if (!fc->connected)
		goto out;

164
165
166
167
	err = -ECONNREFUSED;
	if (fc->conn_error)
		goto out;

168
	req = fuse_request_alloc(npages);
169
	err = -ENOMEM;
170
171
172
	if (!req) {
		if (for_background)
			wake_up(&fc->blocked_waitq);
173
		goto out;
174
	}
175

176
	fuse_req_init_context(fc, req);
Miklos Szeredi's avatar
Miklos Szeredi committed
177
178
179
180
	__set_bit(FR_WAITING, &req->flags);
	if (for_background)
		__set_bit(FR_BACKGROUND, &req->flags);

181
	return req;
182
183

 out:
184
	fuse_drop_waiting(fc);
185
	return ERR_PTR(err);
186
}
187
188
189
190
191

struct fuse_req *fuse_get_req(struct fuse_conn *fc, unsigned npages)
{
	return __fuse_get_req(fc, npages, false);
}
192
EXPORT_SYMBOL_GPL(fuse_get_req);
193

194
195
196
197
198
199
200
struct fuse_req *fuse_get_req_for_background(struct fuse_conn *fc,
					     unsigned npages)
{
	return __fuse_get_req(fc, npages, true);
}
EXPORT_SYMBOL_GPL(fuse_get_req_for_background);

201
202
203
204
205
206
207
208
209
210
211
212
/*
 * Return request in fuse_file->reserved_req.  However that may
 * currently be in use.  If that is the case, wait for it to become
 * available.
 */
static struct fuse_req *get_reserved_req(struct fuse_conn *fc,
					 struct file *file)
{
	struct fuse_req *req = NULL;
	struct fuse_file *ff = file->private_data;

	do {
213
		wait_event(fc->reserved_req_waitq, ff->reserved_req);
214
215
216
217
		spin_lock(&fc->lock);
		if (ff->reserved_req) {
			req = ff->reserved_req;
			ff->reserved_req = NULL;
Al Viro's avatar
Al Viro committed
218
			req->stolen_file = get_file(file);
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
		}
		spin_unlock(&fc->lock);
	} while (!req);

	return req;
}

/*
 * Put stolen request back into fuse_file->reserved_req
 */
static void put_reserved_req(struct fuse_conn *fc, struct fuse_req *req)
{
	struct file *file = req->stolen_file;
	struct fuse_file *ff = file->private_data;

	spin_lock(&fc->lock);
235
	fuse_request_init(req, req->pages, req->page_descs, req->max_pages);
236
237
	BUG_ON(ff->reserved_req);
	ff->reserved_req = req;
238
	wake_up_all(&fc->reserved_req_waitq);
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
	spin_unlock(&fc->lock);
	fput(file);
}

/*
 * Gets a requests for a file operation, always succeeds
 *
 * This is used for sending the FLUSH request, which must get to
 * userspace, due to POSIX locks which may need to be unlocked.
 *
 * If allocation fails due to OOM, use the reserved request in
 * fuse_file.
 *
 * This is very unlikely to deadlock accidentally, since the
 * filesystem should not have it's own file open.  If deadlock is
 * intentional, it can still be broken by "aborting" the filesystem.
 */
256
257
struct fuse_req *fuse_get_req_nofail_nopages(struct fuse_conn *fc,
					     struct file *file)
258
259
260
261
{
	struct fuse_req *req;

	atomic_inc(&fc->num_waiting);
262
	wait_event(fc->blocked_waitq, fc->initialized);
263
264
	/* Matches smp_wmb() in fuse_set_initialized() */
	smp_rmb();
265
	req = fuse_request_alloc(0);
266
267
268
	if (!req)
		req = get_reserved_req(fc, file);

269
	fuse_req_init_context(fc, req);
Miklos Szeredi's avatar
Miklos Szeredi committed
270
271
	__set_bit(FR_WAITING, &req->flags);
	__clear_bit(FR_BACKGROUND, &req->flags);
272
273
274
	return req;
}

275
void fuse_put_request(struct fuse_conn *fc, struct fuse_req *req)
276
{
277
	if (refcount_dec_and_test(&req->count)) {
Miklos Szeredi's avatar
Miklos Szeredi committed
278
		if (test_bit(FR_BACKGROUND, &req->flags)) {
279
280
281
282
283
284
285
286
287
288
			/*
			 * We get here in the unlikely case that a background
			 * request was allocated but not sent
			 */
			spin_lock(&fc->lock);
			if (!fc->blocked)
				wake_up(&fc->blocked_waitq);
			spin_unlock(&fc->lock);
		}

Miklos Szeredi's avatar
Miklos Szeredi committed
289
290
		if (test_bit(FR_WAITING, &req->flags)) {
			__clear_bit(FR_WAITING, &req->flags);
291
			fuse_drop_waiting(fc);
Miklos Szeredi's avatar
Miklos Szeredi committed
292
		}
293
294
295
296
297

		if (req->stolen_file)
			put_reserved_req(fc, req);
		else
			fuse_request_free(req);
298
299
	}
}
300
EXPORT_SYMBOL_GPL(fuse_put_request);
301

302
303
304
305
306
307
308
309
310
311
312
static unsigned len_args(unsigned numargs, struct fuse_arg *args)
{
	unsigned nbytes = 0;
	unsigned i;

	for (i = 0; i < numargs; i++)
		nbytes += args[i].size;

	return nbytes;
}

313
static u64 fuse_get_unique(struct fuse_iqueue *fiq)
314
{
315
	return ++fiq->reqctr;
316
317
}

318
static void queue_request(struct fuse_iqueue *fiq, struct fuse_req *req)
319
320
321
{
	req->in.h.len = sizeof(struct fuse_in_header) +
		len_args(req->in.numargs, (struct fuse_arg *) req->in.args);
322
	list_add_tail(&req->list, &fiq->pending);
Miklos Szeredi's avatar
Miklos Szeredi committed
323
	wake_up_locked(&fiq->waitq);
324
	kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
325
326
}

327
328
329
void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
		       u64 nodeid, u64 nlookup)
{
330
331
	struct fuse_iqueue *fiq = &fc->iq;

332
333
	forget->forget_one.nodeid = nodeid;
	forget->forget_one.nlookup = nlookup;
334

Miklos Szeredi's avatar
Miklos Szeredi committed
335
	spin_lock(&fiq->waitq.lock);
336
	if (fiq->connected) {
337
338
		fiq->forget_list_tail->next = forget;
		fiq->forget_list_tail = forget;
Miklos Szeredi's avatar
Miklos Szeredi committed
339
		wake_up_locked(&fiq->waitq);
340
		kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
Miklos Szeredi's avatar
Miklos Szeredi committed
341
342
343
	} else {
		kfree(forget);
	}
Miklos Szeredi's avatar
Miklos Szeredi committed
344
	spin_unlock(&fiq->waitq.lock);
345
346
}

347
348
static void flush_bg_queue(struct fuse_conn *fc)
{
349
	while (fc->active_background < fc->max_background &&
350
351
	       !list_empty(&fc->bg_queue)) {
		struct fuse_req *req;
352
		struct fuse_iqueue *fiq = &fc->iq;
353
354
355
356

		req = list_entry(fc->bg_queue.next, struct fuse_req, list);
		list_del(&req->list);
		fc->active_background++;
Miklos Szeredi's avatar
Miklos Szeredi committed
357
		spin_lock(&fiq->waitq.lock);
358
359
		req->in.h.unique = fuse_get_unique(fiq);
		queue_request(fiq, req);
Miklos Szeredi's avatar
Miklos Szeredi committed
360
		spin_unlock(&fiq->waitq.lock);
361
362
363
	}
}

364
365
/*
 * This function is called when a request is finished.  Either a reply
366
 * has arrived or it was aborted (and not yet sent) or some error
367
 * occurred during communication with userspace, or the device file
368
369
370
 * was closed.  The requester thread is woken up (if still waiting),
 * the 'end' callback is called if given, else the reference to the
 * request is released
371
372
373
 */
static void request_end(struct fuse_conn *fc, struct fuse_req *req)
{
Miklos Szeredi's avatar
Miklos Szeredi committed
374
	struct fuse_iqueue *fiq = &fc->iq;
Miklos Szeredi's avatar
Miklos Szeredi committed
375

376
	if (test_and_set_bit(FR_FINISHED, &req->flags))
377
		goto put_request;
Miklos Szeredi's avatar
Miklos Szeredi committed
378

Miklos Szeredi's avatar
Miklos Szeredi committed
379
	spin_lock(&fiq->waitq.lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
380
	list_del_init(&req->intr_entry);
Miklos Szeredi's avatar
Miklos Szeredi committed
381
	spin_unlock(&fiq->waitq.lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
382
383
	WARN_ON(test_bit(FR_PENDING, &req->flags));
	WARN_ON(test_bit(FR_SENT, &req->flags));
Miklos Szeredi's avatar
Miklos Szeredi committed
384
	if (test_bit(FR_BACKGROUND, &req->flags)) {
385
		spin_lock(&fc->lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
386
		clear_bit(FR_BACKGROUND, &req->flags);
387
		if (fc->num_background == fc->max_background) {
388
			fc->blocked = 0;
389
			wake_up(&fc->blocked_waitq);
390
391
392
393
394
395
396
397
398
399
		} else if (!fc->blocked) {
			/*
			 * Wake up next waiter, if any.  It's okay to use
			 * waitqueue_active(), as we've already synced up
			 * fc->blocked with waiters with the wake_up() call
			 * above.
			 */
			if (waitqueue_active(&fc->blocked_waitq))
				wake_up(&fc->blocked_waitq);
		}
400

401
		if (fc->num_background == fc->congestion_threshold && fc->sb) {
402
403
			clear_bdi_congested(fc->sb->s_bdi, BLK_RW_SYNC);
			clear_bdi_congested(fc->sb->s_bdi, BLK_RW_ASYNC);
404
		}
405
		fc->num_background--;
406
407
		fc->active_background--;
		flush_bg_queue(fc);
408
		spin_unlock(&fc->lock);
409
	}
410
	wake_up(&req->waitq);
Miklos Szeredi's avatar
Miklos Szeredi committed
411
412
	if (req->end)
		req->end(fc, req);
413
put_request:
414
	fuse_put_request(fc, req);
415
416
}

417
static void queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req)
418
{
Miklos Szeredi's avatar
Miklos Szeredi committed
419
	spin_lock(&fiq->waitq.lock);
420
421
422
423
	if (test_bit(FR_FINISHED, &req->flags)) {
		spin_unlock(&fiq->waitq.lock);
		return;
	}
424
425
426
427
	if (list_empty(&req->intr_entry)) {
		list_add_tail(&req->intr_entry, &fiq->interrupts);
		wake_up_locked(&fiq->waitq);
	}
Miklos Szeredi's avatar
Miklos Szeredi committed
428
	spin_unlock(&fiq->waitq.lock);
429
	kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
430
431
}

432
static void request_wait_answer(struct fuse_conn *fc, struct fuse_req *req)
433
{
Miklos Szeredi's avatar
Miklos Szeredi committed
434
	struct fuse_iqueue *fiq = &fc->iq;
435
436
	int err;

437
438
	if (!fc->no_interrupt) {
		/* Any signal may interrupt this */
439
		err = wait_event_interruptible(req->waitq,
Miklos Szeredi's avatar
Miklos Szeredi committed
440
					test_bit(FR_FINISHED, &req->flags));
441
		if (!err)
442
443
			return;

Miklos Szeredi's avatar
Miklos Szeredi committed
444
		set_bit(FR_INTERRUPTED, &req->flags);
445
446
		/* matches barrier in fuse_dev_do_read() */
		smp_mb__after_atomic();
Miklos Szeredi's avatar
Miklos Szeredi committed
447
		if (test_bit(FR_SENT, &req->flags))
Miklos Szeredi's avatar
Miklos Szeredi committed
448
			queue_interrupt(fiq, req);
449
450
	}

Miklos Szeredi's avatar
Miklos Szeredi committed
451
	if (!test_bit(FR_FORCE, &req->flags)) {
452
		/* Only fatal signals may interrupt this */
453
		err = wait_event_killable(req->waitq,
Miklos Szeredi's avatar
Miklos Szeredi committed
454
					test_bit(FR_FINISHED, &req->flags));
455
		if (!err)
Miklos Szeredi's avatar
Miklos Szeredi committed
456
457
			return;

Miklos Szeredi's avatar
Miklos Szeredi committed
458
		spin_lock(&fiq->waitq.lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
459
		/* Request is not yet in userspace, bail out */
Miklos Szeredi's avatar
Miklos Szeredi committed
460
		if (test_bit(FR_PENDING, &req->flags)) {
Miklos Szeredi's avatar
Miklos Szeredi committed
461
			list_del(&req->list);
Miklos Szeredi's avatar
Miklos Szeredi committed
462
			spin_unlock(&fiq->waitq.lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
463
464
465
466
			__fuse_put_request(req);
			req->out.h.error = -EINTR;
			return;
		}
Miklos Szeredi's avatar
Miklos Szeredi committed
467
		spin_unlock(&fiq->waitq.lock);
468
	}
469

Miklos Szeredi's avatar
Miklos Szeredi committed
470
471
472
473
	/*
	 * Either request is already in userspace, or it was forced.
	 * Wait it out.
	 */
Miklos Szeredi's avatar
Miklos Szeredi committed
474
	wait_event(req->waitq, test_bit(FR_FINISHED, &req->flags));
475
476
}

477
static void __fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
478
{
479
480
	struct fuse_iqueue *fiq = &fc->iq;

Miklos Szeredi's avatar
Miklos Szeredi committed
481
	BUG_ON(test_bit(FR_BACKGROUND, &req->flags));
Miklos Szeredi's avatar
Miklos Szeredi committed
482
	spin_lock(&fiq->waitq.lock);
483
	if (!fiq->connected) {
Miklos Szeredi's avatar
Miklos Szeredi committed
484
		spin_unlock(&fiq->waitq.lock);
485
		req->out.h.error = -ENOTCONN;
486
	} else {
487
488
		req->in.h.unique = fuse_get_unique(fiq);
		queue_request(fiq, req);
489
490
491
		/* acquire extra reference, since request is still needed
		   after request_end() */
		__fuse_get_request(req);
Miklos Szeredi's avatar
Miklos Szeredi committed
492
		spin_unlock(&fiq->waitq.lock);
493

494
		request_wait_answer(fc, req);
495
496
		/* Pairs with smp_wmb() in request_end() */
		smp_rmb();
497
498
	}
}
499
500
501

void fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
{
Miklos Szeredi's avatar
Miklos Szeredi committed
502
503
504
	__set_bit(FR_ISREPLY, &req->flags);
	if (!test_bit(FR_WAITING, &req->flags)) {
		__set_bit(FR_WAITING, &req->flags);
505
506
		atomic_inc(&fc->num_waiting);
	}
507
508
	__fuse_request_send(fc, req);
}
509
EXPORT_SYMBOL_GPL(fuse_request_send);
510

511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args)
{
	if (fc->minor < 4 && args->in.h.opcode == FUSE_STATFS)
		args->out.args[0].size = FUSE_COMPAT_STATFS_SIZE;

	if (fc->minor < 9) {
		switch (args->in.h.opcode) {
		case FUSE_LOOKUP:
		case FUSE_CREATE:
		case FUSE_MKNOD:
		case FUSE_MKDIR:
		case FUSE_SYMLINK:
		case FUSE_LINK:
			args->out.args[0].size = FUSE_COMPAT_ENTRY_OUT_SIZE;
			break;
		case FUSE_GETATTR:
		case FUSE_SETATTR:
			args->out.args[0].size = FUSE_COMPAT_ATTR_OUT_SIZE;
			break;
		}
	}
	if (fc->minor < 12) {
		switch (args->in.h.opcode) {
		case FUSE_CREATE:
			args->in.args[0].size = sizeof(struct fuse_open_in);
			break;
		case FUSE_MKNOD:
			args->in.args[0].size = FUSE_COMPAT_MKNOD_IN_SIZE;
			break;
		}
	}
}

544
545
546
547
548
549
550
551
552
ssize_t fuse_simple_request(struct fuse_conn *fc, struct fuse_args *args)
{
	struct fuse_req *req;
	ssize_t ret;

	req = fuse_get_req(fc, 0);
	if (IS_ERR(req))
		return PTR_ERR(req);

553
554
555
	/* Needs to be done after fuse_get_req() so that fc->minor is valid */
	fuse_adjust_compat(fc, args);

556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
	req->in.h.opcode = args->in.h.opcode;
	req->in.h.nodeid = args->in.h.nodeid;
	req->in.numargs = args->in.numargs;
	memcpy(req->in.args, args->in.args,
	       args->in.numargs * sizeof(struct fuse_in_arg));
	req->out.argvar = args->out.argvar;
	req->out.numargs = args->out.numargs;
	memcpy(req->out.args, args->out.args,
	       args->out.numargs * sizeof(struct fuse_arg));
	fuse_request_send(fc, req);
	ret = req->out.h.error;
	if (!ret && args->out.argvar) {
		BUG_ON(args->out.numargs != 1);
		ret = req->out.args[0].size;
	}
	fuse_put_request(fc, req);

	return ret;
}

576
577
578
579
580
581
582
/*
 * Called under fc->lock
 *
 * fc->connected must have been checked previously
 */
void fuse_request_send_background_locked(struct fuse_conn *fc,
					 struct fuse_req *req)
583
{
Miklos Szeredi's avatar
Miklos Szeredi committed
584
585
586
	BUG_ON(!test_bit(FR_BACKGROUND, &req->flags));
	if (!test_bit(FR_WAITING, &req->flags)) {
		__set_bit(FR_WAITING, &req->flags);
587
588
		atomic_inc(&fc->num_waiting);
	}
Miklos Szeredi's avatar
Miklos Szeredi committed
589
	__set_bit(FR_ISREPLY, &req->flags);
590
	fc->num_background++;
591
	if (fc->num_background == fc->max_background)
592
		fc->blocked = 1;
Jan Kara's avatar
Jan Kara committed
593
	if (fc->num_background == fc->congestion_threshold && fc->sb) {
594
595
		set_bdi_congested(fc->sb->s_bdi, BLK_RW_SYNC);
		set_bdi_congested(fc->sb->s_bdi, BLK_RW_ASYNC);
596
597
598
599
600
	}
	list_add_tail(&req->list, &fc->bg_queue);
	flush_bg_queue(fc);
}

601
void fuse_request_send_background(struct fuse_conn *fc, struct fuse_req *req)
602
{
603
	BUG_ON(!req->end);
604
	spin_lock(&fc->lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
605
	if (fc->connected) {
606
		fuse_request_send_background_locked(fc, req);
607
		spin_unlock(&fc->lock);
608
	} else {
609
		spin_unlock(&fc->lock);
610
		req->out.h.error = -ENOTCONN;
611
612
		req->end(fc, req);
		fuse_put_request(fc, req);
613
614
	}
}
615
EXPORT_SYMBOL_GPL(fuse_request_send_background);
616

Miklos Szeredi's avatar
Miklos Szeredi committed
617
618
619
620
static int fuse_request_send_notify_reply(struct fuse_conn *fc,
					  struct fuse_req *req, u64 unique)
{
	int err = -ENODEV;
621
	struct fuse_iqueue *fiq = &fc->iq;
Miklos Szeredi's avatar
Miklos Szeredi committed
622

Miklos Szeredi's avatar
Miklos Szeredi committed
623
	__clear_bit(FR_ISREPLY, &req->flags);
Miklos Szeredi's avatar
Miklos Szeredi committed
624
	req->in.h.unique = unique;
Miklos Szeredi's avatar
Miklos Szeredi committed
625
	spin_lock(&fiq->waitq.lock);
626
	if (fiq->connected) {
627
		queue_request(fiq, req);
Miklos Szeredi's avatar
Miklos Szeredi committed
628
629
		err = 0;
	}
Miklos Szeredi's avatar
Miklos Szeredi committed
630
	spin_unlock(&fiq->waitq.lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
631
632
633
634

	return err;
}

635
636
void fuse_force_forget(struct file *file, u64 nodeid)
{
637
	struct inode *inode = file_inode(file);
638
639
640
641
642
643
	struct fuse_conn *fc = get_fuse_conn(inode);
	struct fuse_req *req;
	struct fuse_forget_in inarg;

	memset(&inarg, 0, sizeof(inarg));
	inarg.nlookup = 1;
644
	req = fuse_get_req_nofail_nopages(fc, file);
645
646
647
648
649
	req->in.h.opcode = FUSE_FORGET;
	req->in.h.nodeid = nodeid;
	req->in.numargs = 1;
	req->in.args[0].size = sizeof(inarg);
	req->in.args[0].value = &inarg;
Miklos Szeredi's avatar
Miklos Szeredi committed
650
	__clear_bit(FR_ISREPLY, &req->flags);
651
652
653
	__fuse_request_send(fc, req);
	/* ignore errors */
	fuse_put_request(fc, req);
654
655
}

656
657
658
/*
 * Lock the request.  Up to the next unlock_request() there mustn't be
 * anything that could cause a page-fault.  If the request was already
659
 * aborted bail out.
660
 */
661
static int lock_request(struct fuse_req *req)
662
663
664
{
	int err = 0;
	if (req) {
665
		spin_lock(&req->waitq.lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
666
		if (test_bit(FR_ABORTED, &req->flags))
667
668
			err = -ENOENT;
		else
Miklos Szeredi's avatar
Miklos Szeredi committed
669
			set_bit(FR_LOCKED, &req->flags);
670
		spin_unlock(&req->waitq.lock);
671
672
673
674
675
	}
	return err;
}

/*
Miklos Szeredi's avatar
Miklos Szeredi committed
676
677
 * Unlock request.  If it was aborted while locked, caller is responsible
 * for unlocking and ending the request.
678
 */
679
static int unlock_request(struct fuse_req *req)
680
{
Miklos Szeredi's avatar
Miklos Szeredi committed
681
	int err = 0;
682
	if (req) {
683
		spin_lock(&req->waitq.lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
684
		if (test_bit(FR_ABORTED, &req->flags))
Miklos Szeredi's avatar
Miklos Szeredi committed
685
686
			err = -ENOENT;
		else
Miklos Szeredi's avatar
Miklos Szeredi committed
687
			clear_bit(FR_LOCKED, &req->flags);
688
		spin_unlock(&req->waitq.lock);
689
	}
Miklos Szeredi's avatar
Miklos Szeredi committed
690
	return err;
691
692
693
694
695
}

struct fuse_copy_state {
	int write;
	struct fuse_req *req;
696
	struct iov_iter *iter;
697
698
699
	struct pipe_buffer *pipebufs;
	struct pipe_buffer *currbuf;
	struct pipe_inode_info *pipe;
700
701
702
	unsigned long nr_segs;
	struct page *pg;
	unsigned len;
703
	unsigned offset;
704
	unsigned move_pages:1;
705
706
};

707
static void fuse_copy_init(struct fuse_copy_state *cs, int write,
708
			   struct iov_iter *iter)
709
710
711
{
	memset(cs, 0, sizeof(*cs));
	cs->write = write;
712
	cs->iter = iter;
713
714
715
}

/* Unmap and put previous page of userspace buffer */
716
static void fuse_copy_finish(struct fuse_copy_state *cs)
717
{
718
719
720
	if (cs->currbuf) {
		struct pipe_buffer *buf = cs->currbuf;

721
		if (cs->write)
722
			buf->len = PAGE_SIZE - cs->len;
723
		cs->currbuf = NULL;
724
	} else if (cs->pg) {
725
726
727
728
729
730
		if (cs->write) {
			flush_dcache_page(cs->pg);
			set_page_dirty_lock(cs->pg);
		}
		put_page(cs->pg);
	}
731
	cs->pg = NULL;
732
733
734
735
736
737
738
739
}

/*
 * Get another pagefull of userspace buffer, and map it to kernel
 * address space, and lock request
 */
static int fuse_copy_fill(struct fuse_copy_state *cs)
{
740
	struct page *page;
741
742
	int err;

743
	err = unlock_request(cs->req);
Miklos Szeredi's avatar
Miklos Szeredi committed
744
745
746
	if (err)
		return err;

747
	fuse_copy_finish(cs);
748
749
750
	if (cs->pipebufs) {
		struct pipe_buffer *buf = cs->pipebufs;

751
		if (!cs->write) {
752
			err = pipe_buf_confirm(cs->pipe, buf);
753
754
755
756
757
			if (err)
				return err;

			BUG_ON(!cs->nr_segs);
			cs->currbuf = buf;
758
759
			cs->pg = buf->page;
			cs->offset = buf->offset;
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
			cs->len = buf->len;
			cs->pipebufs++;
			cs->nr_segs--;
		} else {
			if (cs->nr_segs == cs->pipe->buffers)
				return -EIO;

			page = alloc_page(GFP_HIGHUSER);
			if (!page)
				return -ENOMEM;

			buf->page = page;
			buf->offset = 0;
			buf->len = 0;

			cs->currbuf = buf;
776
777
			cs->pg = page;
			cs->offset = 0;
778
779
780
781
			cs->len = PAGE_SIZE;
			cs->pipebufs++;
			cs->nr_segs++;
		}
782
	} else {
783
784
		size_t off;
		err = iov_iter_get_pages(cs->iter, &page, PAGE_SIZE, 1, &off);
785
786
		if (err < 0)
			return err;
787
788
789
		BUG_ON(!err);
		cs->len = err;
		cs->offset = off;
790
		cs->pg = page;
791
		iov_iter_advance(cs->iter, err);
792
793
	}

794
	return lock_request(cs->req);
795
796
797
}

/* Do as much copy to/from userspace buffer as we can */
798
static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)
799
800
801
{
	unsigned ncpy = min(*size, cs->len);
	if (val) {
802
803
804
		void *pgaddr = kmap_atomic(cs->pg);
		void *buf = pgaddr + cs->offset;

805
		if (cs->write)
806
			memcpy(buf, *val, ncpy);
807
		else
808
809
810
			memcpy(*val, buf, ncpy);

		kunmap_atomic(pgaddr);
811
812
813
814
		*val += ncpy;
	}
	*size -= ncpy;
	cs->len -= ncpy;
815
	cs->offset += ncpy;
816
817
818
	return ncpy;
}

819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
static int fuse_check_page(struct page *page)
{
	if (page_mapcount(page) ||
	    page->mapping != NULL ||
	    page_count(page) != 1 ||
	    (page->flags & PAGE_FLAGS_CHECK_AT_PREP &
	     ~(1 << PG_locked |
	       1 << PG_referenced |
	       1 << PG_uptodate |
	       1 << PG_lru |
	       1 << PG_active |
	       1 << PG_reclaim))) {
		printk(KERN_WARNING "fuse: trying to steal weird page\n");
		printk(KERN_WARNING "  page=%p index=%li flags=%08lx, count=%i, mapcount=%i, mapping=%p\n", page, page->index, page->flags, page_count(page), page_mapcount(page), page->mapping);
		return 1;
	}
	return 0;
}

static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
{
	int err;
	struct page *oldpage = *pagep;
	struct page *newpage;
	struct pipe_buffer *buf = cs->pipebufs;

845
	err = unlock_request(cs->req);
Miklos Szeredi's avatar
Miklos Szeredi committed
846
847
848
	if (err)
		return err;

849
850
	fuse_copy_finish(cs);

851
	err = pipe_buf_confirm(cs->pipe, buf);
852
853
854
855
856
857
858
859
860
861
862
863
	if (err)
		return err;

	BUG_ON(!cs->nr_segs);
	cs->currbuf = buf;
	cs->len = buf->len;
	cs->pipebufs++;
	cs->nr_segs--;

	if (cs->len != PAGE_SIZE)
		goto out_fallback;

864
	if (pipe_buf_steal(cs->pipe, buf) != 0)
865
866
867
868
		goto out_fallback;

	newpage = buf->page;

869
870
	if (!PageUptodate(newpage))
		SetPageUptodate(newpage);
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889

	ClearPageMappedToDisk(newpage);

	if (fuse_check_page(newpage) != 0)
		goto out_fallback_unlock;

	/*
	 * This is a new and locked page, it shouldn't be mapped or
	 * have any special flags on it
	 */
	if (WARN_ON(page_mapped(oldpage)))
		goto out_fallback_unlock;
	if (WARN_ON(page_has_private(oldpage)))
		goto out_fallback_unlock;
	if (WARN_ON(PageDirty(oldpage) || PageWriteback(oldpage)))
		goto out_fallback_unlock;
	if (WARN_ON(PageMlocked(oldpage)))
		goto out_fallback_unlock;

890
	err = replace_page_cache_page(oldpage, newpage, GFP_KERNEL);
891
	if (err) {
892
893
		unlock_page(newpage);
		return err;
894
	}
895

896
	get_page(newpage);
897
898
899
900
901

	if (!(buf->flags & PIPE_BUF_FLAG_LRU))
		lru_cache_add_file(newpage);

	err = 0;
902
	spin_lock(&cs->req->waitq.lock);
Miklos Szeredi's avatar
Miklos Szeredi committed
903
	if (test_bit(FR_ABORTED, &cs->req->flags))
904
905
906
		err = -ENOENT;
	else
		*pagep = newpage;
907
	spin_unlock(&cs->req->waitq.lock);
908
909
910

	if (err) {
		unlock_page(newpage);
911
		put_page(newpage);
912
913
914
915
		return err;
	}

	unlock_page(oldpage);
916
	put_page(oldpage);
917
918
919
920
921
922
923
	cs->len = 0;

	return 0;

out_fallback_unlock:
	unlock_page(newpage);
out_fallback:
924
925
	cs->pg = buf->page;
	cs->offset = buf->offset;
926

927
	err = lock_request(cs->req);
928
929
930
931
932
933
	if (err)
		return err;

	return 1;
}

934
935
936
937
static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
			 unsigned offset, unsigned count)
{
	struct pipe_buffer *buf;
Miklos Szeredi's avatar
Miklos Szeredi committed
938
	int err;
939
940
941
942

	if (cs->nr_segs == cs->pipe->buffers)
		return -EIO;

943
	err = unlock_request(cs->req);
Miklos Szeredi's avatar
Miklos Szeredi committed
944
945
946
	if (err)
		return err;

947
948
949
	fuse_copy_finish(cs);

	buf = cs->pipebufs;
950
	get_page(page);
951
952
953
954
955
956
957
958
959
960
961
	buf->page = page;
	buf->offset = offset;
	buf->len = count;

	cs->pipebufs++;
	cs->nr_segs++;
	cs->len = 0;

	return 0;
}

962
963
964
965
/*
 * Copy a page in the request to/from the userspace buffer.  Must be
 * done atomically
 */
966
static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
967
			  unsigned offset, unsigned count, int zeroing)
968
{
969
970
971
	int err;
	struct page *page = *pagep;

972
973
974
	if (page && zeroing && count < PAGE_SIZE)
		clear_highpage(page);

975
	while (count) {
976
977
978
		if (cs->write && cs->pipebufs && page) {
			return fuse_ref_page(cs, page, offset, count);
		} else if (!cs->len) {
979
980
981
982
983
984
985
986
987
988
			if (cs->move_pages && page &&
			    offset == 0 && count == PAGE_SIZE) {
				err = fuse_try_move_page(cs, pagep);
				if (err <= 0)
					return err;
			} else {
				err = fuse_copy_fill(cs);
				if (err)
					return err;
			}
Miklos Szeredi's avatar
Miklos Szeredi committed
989
		}
990
		if (page) {
991
			void *mapaddr = kmap_atomic(page);
992
993
			void *buf = mapaddr + offset;
			offset += fuse_copy_do(cs, &buf, &count);
994
			kunmap_atomic(mapaddr);
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
		} else
			offset += fuse_copy_do(cs, NULL, &count);
	}
	if (page && !cs->write)
		flush_dcache_page(page);
	return 0;
}

/* Copy pages in the request to/from userspace buffer */
static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes,
			   int zeroing)
{
	unsigned i;
	struct fuse_req *req = cs->req;

	for (i = 0; i < req->num_pages && (nbytes || zeroing); i++) {
1011
		int err;
1012
1013
		unsigned offset = req->page_descs[i].offset;
		unsigned count = min(nbytes, req->page_descs[i].length);
1014
1015
1016

		err = fuse_copy_page(cs, &req->pages[i], offset, count,
				     zeroing);
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
		if (err)
			return err;

		nbytes -= count;
	}
	return 0;
}

/* Copy a single argument in the request to/from userspace buffer */
static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size)
{
	while (size) {
Miklos Szeredi's avatar
Miklos Szeredi committed
1029
1030
1031
1032
1033
		if (!cs->len) {
			int err = fuse_copy_fill(cs);
			if (err)
				return err;
		}
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
		fuse_copy_do(cs, &val, &size);
	}
	return 0;
}

/* Copy request arguments to/from userspace buffer */
static int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs,
			  unsigned argpages, struct fuse_arg *args,
			  int zeroing)
{
	int err = 0;
	unsigned i;

	for (i = 0; !err && i < numargs; i++)  {
		struct fuse_arg *arg = &args[i];
		if (i =