poll.c 14.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
/*
 * SPDX-License-Identifier: GPL-2.0
 *
 * Copyright (C) 2018 Philippe Gerum  <rpm@xenomai.org>
 */

#include <linux/types.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include <linux/rbtree.h>
#include <linux/poll.h>
#include <linux/module.h>
13
#include <linux/spinlock.h>
14
15
16
17
18
19
20
21
#include <evl/file.h>
#include <evl/thread.h>
#include <evl/memory.h>
#include <evl/poll.h>
#include <evl/sched.h>
#include <evl/flag.h>
#include <evl/mutex.h>
#include <asm/evl/syscall.h>
22

23
24
25
26
struct poll_group {
	struct rb_root item_index;  /* struct poll_item */
	struct list_head item_list; /* struct poll_item */
	struct list_head waiter_list; /* struct poll_waiter */
27
	evl_spinlock_t wait_lock;
28
	struct evl_file efile;
29
30
	struct evl_kmutex item_lock;
	int nr_items;
31
32
33
	unsigned int generation;
};

34
struct poll_item {
35
36
	unsigned int fd;
	int events_polled;
37
38
39
40
41
42
43
	struct rb_node rb;	    /* in group->item_index */
	struct list_head next;	    /* in group->item_list */
};

struct poll_waiter {
	struct evl_flag flag;
	struct list_head next;
44
45
};

46
/* Maximum nesting depth (poll group watching other group(s)) */
47
48
#define POLLER_NEST_MAX  4

49
static const struct file_operations poll_fops;
50

51
void evl_poll_watch(struct evl_poll_head *head,
52
53
		struct oob_poll_wait *wait,
		void (*unwatch)(struct file *filp))
54
55
56
57
58
{
	struct evl_poll_watchpoint *wpt;
	unsigned long flags;

	wpt = container_of(wait, struct evl_poll_watchpoint, wait);
59
	/* Add to driver's poll head. */
60
	evl_spin_lock_irqsave(&head->lock, flags);
61
	wpt->head = head;
62
	wpt->events_received = 0;
63
	wpt->unwatch = unwatch;
64
	list_add(&wait->next, &head->watchpoints);
65
	evl_spin_unlock_irqrestore(&head->lock, flags);
66
67
68
69
}
EXPORT_SYMBOL_GPL(evl_poll_watch);

void __evl_signal_poll_events(struct evl_poll_head *head,
70
			int events)
71
{
72
	struct evl_poll_watchpoint *wpt;
73
74
75
	unsigned long flags;
	int ready;

76
	evl_spin_lock_irqsave(&head->lock, flags);
77

78
	list_for_each_entry(wpt, &head->watchpoints, wait.next) {
79
		ready = events & wpt->events_polled;
80
81
82
		if (ready) {
			wpt->events_received |= ready;
			evl_raise_flag_nosched(wpt->flag);
83
84
85
		}
	}

86
	evl_spin_unlock_irqrestore(&head->lock, flags);
87
88
89
}
EXPORT_SYMBOL_GPL(__evl_signal_poll_events);

90
91
92
93
94
95
96
97
98
void evl_drop_poll_table(struct evl_thread *thread)
{
	struct evl_poll_watchpoint *table;

	table = thread->poll_context.table;
	if (table)
		evl_free(table);
}

99
static inline
100
int index_item(struct rb_root *root, struct poll_item *item)
101
102
{
	struct rb_node **rbp, *parent = NULL;
103
	struct poll_item *tmp;
104
105
106

	rbp = &root->rb_node;
	while (*rbp) {
107
		tmp = rb_entry(*rbp, struct poll_item, rb);
108
		parent = *rbp;
109
		if (item->fd < tmp->fd)
110
			rbp = &(*rbp)->rb_left;
111
		else if (item->fd > tmp->fd)
112
113
114
115
116
			rbp = &(*rbp)->rb_right;
		else
			return -EEXIST;
	}

117
118
	rb_link_node(&item->rb, parent, rbp);
	rb_insert_color(&item->rb, root);
119
120
121
122

	return 0;
}

123
static inline void new_generation(struct poll_group *group)
124
{
125
126
	if (++group->generation == 0) /* Keep zero for init state. */
		group->generation = 1;
127
128
}

129
130
static int check_no_loop_deeper(struct poll_group *origin,
				struct poll_item *item,
131
				int depth)
132
{
133
134
135
	struct poll_group *group;
	struct poll_item *_item;
	struct evl_file *efilp;
136
137
138
139
	struct file *filp;
	int ret = 0;

	if (depth >= POLLER_NEST_MAX)
140
		return -ELOOP;
141

142
143
	efilp = evl_get_file(item->fd);
	if (efilp == NULL)
144
145
		return 0;

146
147
148
149
150
151
	filp = efilp->filp;
	if (filp->f_op != &poll_fops)
		goto out;

	group = filp->private_data;
	if (group == origin) {
152
		ret = -ELOOP;
153
154
		goto out;
	}
155

156
	evl_lock_kmutex(&group->item_lock);
157

158
159
	list_for_each_entry(_item, &group->item_list, next) {
		ret = check_no_loop_deeper(origin, _item, depth + 1);
160
161
162
163
		if (ret)
			break;
	}

164
165
166
	evl_unlock_kmutex(&group->item_lock);
out:
	evl_put_file(efilp);
167
168
169
170

	return ret;
}

171
172
static int check_no_loop(struct poll_group *group,
			struct poll_item *item)
173
{
174
	return check_no_loop_deeper(group, item, 0);
175
176
}

177
static int add_item(struct file *filp, struct poll_group *group,
178
		struct evl_poll_ctlreq *creq)
179
{
180
181
182
183
184
185
	struct poll_item *item;
	struct evl_file *efilp;
	int ret, events;

	item = evl_alloc(sizeof(*item));
	if (item == NULL)
186
187
		return -ENOMEM;

188
	item->fd = creq->fd;
189
190
	events = creq->events & ~POLLNVAL;
	item->events_polled = events | POLLERR | POLLHUP;
191
192
193

	efilp = evl_get_file(creq->fd);
	if (efilp == NULL) {
194
		ret = -EBADF;
195
196
197
		goto fail_get;
	}

198
	evl_lock_kmutex(&group->item_lock);
199

200
	/* Check for cyclic deps. */
201
	ret = check_no_loop(group, item);
202
203
204
	if (ret)
		goto fail_add;

205
	ret = index_item(&group->item_index, item);
206
207
208
	if (ret)
		goto fail_add;

209
210
211
	list_add(&item->next, &group->item_list);
	group->nr_items++;
	new_generation(group);
212

213
214
	evl_unlock_kmutex(&group->item_lock);
	evl_put_file(efilp);
215
216
217
218

	return 0;

fail_add:
219
220
	evl_unlock_kmutex(&group->item_lock);
	evl_put_file(efilp);
221
fail_get:
222
	evl_free(item);
223
224
225
226

	return ret;
}

227
228
static struct poll_item *
lookup_item(struct rb_root *root, unsigned int fd)
229
{
230
	struct poll_item *item;
231
232
233
234
	struct rb_node *rb;

	rb = root->rb_node;
	while (rb) {
235
236
		item = rb_entry(rb, struct poll_item, rb);
		if (fd < item->fd)
237
			rb = rb->rb_left;
238
		else if (fd > item->fd)
239
240
			rb = rb->rb_right;
		else
241
			return item;
242
243
244
245
246
	}

	return NULL;
}

247
static int del_item(struct poll_group *group,
248
		struct evl_poll_ctlreq *creq)
249
{
250
	struct poll_item *item;
251

252
	evl_lock_kmutex(&group->item_lock);
253

254
255
256
	item = lookup_item(&group->item_index, creq->fd);
	if (item == NULL) {
		evl_unlock_kmutex(&group->item_lock);
257
		return -ENOENT;
258
259
	}

260
261
262
263
	rb_erase(&item->rb, &group->item_index);
	list_del(&item->next);
	group->nr_items--;
	new_generation(group);
264

265
	evl_unlock_kmutex(&group->item_lock);
266

267
	evl_free(item);
268
269
270
271

	return 0;
}

272
273
274
275
276
277
278
/* fdt_lock held, irqs off. */
void evl_drop_watchpoints(struct list_head *drop_list)
{
	struct evl_poll_watchpoint *wpt;
	struct evl_poll_node *node;

	/*
279
280
281
282
283
284
285
286
287
	 * Drop the watchpoints attached to a closed file descriptor
	 * upon release from inband. A watchpoint found in @drop_list
	 * was registered via a call to evl_watch_fd() from
	 * wait_events() but not unregistered by calling
	 * evl_ignore_fd() from clear_wait() yet, so we know it is
	 * still valid. Since a polled EVL fd has to be passed to this
	 * routine before the file it references can be dismantled, we
	 * may keep and use a direct pointer to this file in the
	 * watchpoint struct until we return.
288
289
290
	 */
	list_for_each_entry(node, drop_list, next) {
		wpt = container_of(node, struct evl_poll_watchpoint, node);
291
		evl_spin_lock(&wpt->head->lock);
292
		wpt->events_received |= POLLNVAL;
293
294
		if (wpt->unwatch)
			wpt->unwatch(wpt->filp);
295
		evl_raise_flag_nosched(wpt->flag);
296
		evl_spin_unlock(&wpt->head->lock);
297
		wpt->filp = NULL;
298
299
300
	}
}

301
static inline
302
int mod_item(struct poll_group *group,
303
	struct evl_poll_ctlreq *creq)
304
{
305
306
307
	struct poll_item *item;
	int events;

308
	events = creq->events & ~POLLNVAL;
309

310
	evl_lock_kmutex(&group->item_lock);
311

312
313
314
	item = lookup_item(&group->item_index, creq->fd);
	if (item == NULL) {
		evl_unlock_kmutex(&group->item_lock);
315
		return -ENOENT;
316
317
	}

318
	item->events_polled = events | POLLERR | POLLHUP;
319
	new_generation(group);
320

321
	evl_unlock_kmutex(&group->item_lock);
322
323
324
325
326

	return 0;
}

static inline
327
int setup_item(struct file *filp, struct poll_group *group,
328
	struct evl_poll_ctlreq *creq)
329
330
331
332
{
	int ret;

	switch (creq->action) {
333
	case EVL_POLL_CTLADD:
334
		ret = add_item(filp, group, creq);
335
		break;
336
	case EVL_POLL_CTLDEL:
337
		ret = del_item(group, creq);
338
		break;
339
	case EVL_POLL_CTLMOD:
340
		ret = mod_item(group, creq);
341
		break;
342
343
	default:
		ret = -EINVAL;
344
345
	}

346
	return ret;
347
348
}

349
static int collect_events(struct poll_group *group,
350
351
			struct evl_poll_event __user *u_set,
			int maxevents, struct evl_flag *flag)
352
{
353
	struct evl_thread *curr = evl_current();
354
355
356
357
	struct evl_poll_watchpoint *wpt, *table;
	int ret, n, nr, count = 0, ready;
	struct evl_poll_event ev;
	unsigned int generation;
358
359
	struct poll_item *item;
	struct evl_file *efilp;
360
361
	struct file *filp;

362
	evl_lock_kmutex(&group->item_lock);
363

364
	nr = group->nr_items;
365
	if (nr == 0) {
366
		evl_unlock_kmutex(&group->item_lock);
367
368
369
370
		return -EINVAL;
	}

	/*
371
	 * Check whether the registered items are in sync with the
372
373
374
375
	 * caller's registered watchpoints (if any). Go polling
	 * directly using those watchpoints if so, otherwise resync.
	 */
	table = curr->poll_context.table;
376
377
378
	if (flag == NULL)
		goto collect;

379
380
	generation = group->generation;
	if (likely(generation == curr->poll_context.generation))
381
382
383
384
		goto collect;

	/* Need to resync. */
	do {
385
386
		generation = group->generation;
		evl_unlock_kmutex(&group->item_lock);
387
		evl_drop_poll_table(curr);
388
389
390
391
392
393
394
		table = evl_alloc(sizeof(*wpt) * nr);
		if (table == NULL) {
			curr->poll_context.nr = 0;
			curr->poll_context.table = NULL;
			curr->poll_context.generation = 0;
			return -ENOMEM;
		}
395
396
		evl_lock_kmutex(&group->item_lock);
	} while (generation != group->generation);
397
398
399
400
401

	curr->poll_context.table = table;
	curr->poll_context.nr = nr;
	curr->poll_context.generation = generation;

402
	/* Build the poll table. */
403
	wpt = table;
404
405
406
	list_for_each_entry(item, &group->item_list, next) {
		wpt->fd = item->fd;
		wpt->events_polled = item->events_polled;
407
408
409
410
		wpt++;
	}

collect:
411
	evl_unlock_kmutex(&group->item_lock);
412
413

	for (n = 0, wpt = table; n < nr; n++, wpt++) {
414
415
		if (flag) {
			wpt->flag = flag;
416
			INIT_LIST_HEAD(&wpt->wait.next);
417
418
			/* If oob_poll() is absent, default to all events ready. */
			ready = POLLIN|POLLOUT|POLLRDNORM|POLLWRNORM;
419
420
421
422
			efilp = evl_watch_fd(wpt->fd, &wpt->node);
			if (efilp == NULL)
				goto stale;
			filp = efilp->filp;
423
			wpt->filp = filp;
424
425
			if (filp->f_op->oob_poll)
				ready = filp->f_op->oob_poll(filp, &wpt->wait);
426
			evl_put_file(efilp);
427
		} else
428
			ready = wpt->events_received;
429

430
		ready &= wpt->events_polled | POLLNVAL;
431
		if (ready) {
432
			ev.fd = wpt->fd;
433
			ev.events = ready;
434
			ret = raw_copy_to_user(u_set, &ev, sizeof(ev));
435
436
			if (ret)
				return -EFAULT;
437
			u_set++;
438
439
440
441
442
443
			if (++count >= maxevents)
				break;
		}
	}

	return count;
444
445
446
447
448
449
450
451
452
453
stale:
	/*
	 * We have a stale fd in the table, force regeneration next
	 * time we collect events then bail out on error.
	 */
	evl_lock_kmutex(&group->item_lock);
	new_generation(group);
	evl_unlock_kmutex(&group->item_lock);

	return -EBADF;
454
455
456
457
}

static inline void clear_wait(void)
{
458
	struct evl_thread *curr = evl_current();
459
460
461
462
463
464
465
466
	struct evl_poll_watchpoint *wpt;
	unsigned long flags;
	int n;

	/*
	 * Current stopped waiting for events, remove the watchpoints
	 * we have been monitoring so far from their poll heads.
	 * wpt->head->lock serializes with __evl_signal_poll_events().
467
	 * Any watchpoint which does not bear the POLLNVAL bit is
468
	 * monitoring a still valid file by construction.
469
470
471
	 */
	for (n = 0, wpt = curr->poll_context.table;
	     n < curr->poll_context.nr; n++, wpt++) {
472
473
474
		evl_ignore_fd(&wpt->node);
		/* Remove from driver's poll head. */
		if (!list_empty(&wpt->wait.next)) {
475
			evl_spin_lock_irqsave(&wpt->head->lock, flags);
476
			list_del(&wpt->wait.next);
477
			if (!(wpt->events_received & POLLNVAL) && wpt->unwatch)
478
				wpt->unwatch(wpt->filp);
479
			evl_spin_unlock_irqrestore(&wpt->head->lock, flags);
480
		}
481
482
483
484
485
	}
}

static inline
int wait_events(struct file *filp,
486
		struct poll_group *group,
487
		struct evl_poll_waitreq *wreq)
488
{
489
	struct poll_waiter waiter;
490
	enum evl_tmode tmode;
491
	unsigned long flags;
492
	ktime_t timeout;
493
	int ret, count;
494

495
	if (wreq->nrset < 0)
496
497
498
499
500
		return -EINVAL;

	if ((unsigned long)wreq->timeout.tv_nsec >= ONE_BILLION)
		return -EINVAL;

501
	if (wreq->nrset == 0)
502
503
		return 0;

504
	evl_init_flag(&waiter.flag);
505

506
	count = collect_events(group, wreq->pollset, wreq->nrset, &waiter.flag);
507
	if (count > 0 || (count == -EFAULT || count == -EBADF))
508
509
		goto unwait;
	if (count < 0)
510
		goto out;
511
512
513
514
515
516
517
518

	if (filp->f_flags & O_NONBLOCK) {
		count = -EAGAIN;
		goto unwait;
	}

	timeout = timespec_to_ktime(wreq->timeout);
	tmode = timeout ? EVL_ABS : EVL_REL;
519

520
	evl_spin_lock_irqsave(&group->wait_lock, flags);
521
	list_add(&waiter.next, &group->waiter_list);
522
	evl_spin_unlock_irqrestore(&group->wait_lock, flags);
523
	ret = evl_wait_flag_timeout(&waiter.flag, timeout, tmode);
524
	evl_spin_lock_irqsave(&group->wait_lock, flags);
525
	list_del(&waiter.next);
526
	evl_spin_unlock_irqrestore(&group->wait_lock, flags);
527
528
529
530

	count = ret;
	if (count == 0)	/* Re-collect events after successful wait. */
		count = collect_events(group, wreq->pollset,
531
				wreq->nrset, NULL);
532
533
unwait:
	clear_wait();
534
out:
535
	evl_destroy_flag(&waiter.flag);
536
537
538
539

	return count;
}

540
static int poll_open(struct inode *inode, struct file *filp)
541
{
542
	struct poll_group *group;
543
544
	int ret;

545
546
	group = kzalloc(sizeof(*group), GFP_KERNEL);
	if (group == NULL)
547
548
		return -ENOMEM;

549
	ret = evl_open_file(&group->efile, filp);
550
	if (ret) {
551
		kfree(group);
552
		return ret;
553
	}
554

555
556
557
558
	group->item_index = RB_ROOT;
	INIT_LIST_HEAD(&group->item_list);
	INIT_LIST_HEAD(&group->waiter_list);
	evl_init_kmutex(&group->item_lock);
559
	evl_spin_lock_init(&group->wait_lock);
560
	filp->private_data = group;
561
562
563
564

	return ret;
}

565
static inline void flush_items(struct poll_group *group)
566
{
567
	struct poll_item *item, *n;
568

569
570
	list_for_each_entry_safe(item, n, &group->item_list, next)
		evl_free(item);
571
572
573
574
}

static int poll_release(struct inode *inode, struct file *filp)
{
575
576
577
578
	struct poll_group *group = filp->private_data;
	struct poll_waiter *waiter;
	unsigned long flags;

579
	evl_spin_lock_irqsave(&group->wait_lock, flags);
580
581
	list_for_each_entry(waiter, &group->waiter_list, next)
		evl_flush_flag_nosched(&waiter->flag, T_RMID);
582
	evl_spin_unlock_irqrestore(&group->wait_lock, flags);
583
	evl_schedule();
584

585
586
587
	flush_items(group);
	evl_release_file(&group->efile);
	kfree(group);
588
589
590
591
592

	return 0;
}

static long poll_oob_ioctl(struct file *filp, unsigned int cmd,
593
			unsigned long arg)
594
{
595
	struct poll_group *group = filp->private_data;
596
597
	struct evl_poll_waitreq wreq, __user *u_wreq;
	struct evl_poll_ctlreq creq, __user *u_creq;
598
599
600
601
602
603
604
605
	int ret;

	switch (cmd) {
	case EVL_POLIOC_CTL:
		u_creq = (typeof(u_creq))arg;
		ret = raw_copy_from_user(&creq, u_creq, sizeof(creq));
		if (ret)
			return -EFAULT;
606
		ret = setup_item(filp, group, &creq);
607
608
609
610
611
612
		break;
	case EVL_POLIOC_WAIT:
		u_wreq = (typeof(u_wreq))arg;
		ret = raw_copy_from_user(&wreq, u_wreq, sizeof(wreq));
		if (ret)
			return -EFAULT;
613
		ret = wait_events(filp, group, &wreq);
614
615
616
		if (ret < 0)
			return ret;
		if (raw_put_user(ret, &u_wreq->nrset))
617
618
619
620
621
622
623
624
625
626
			return -EFAULT;
		ret = 0;
		break;
	default:
		ret = -ENOTTY;
	}

	return ret;
}

627
628
629
630
static const struct file_operations poll_fops = {
	.open		= poll_open,
	.release	= poll_release,
	.oob_ioctl	= poll_oob_ioctl,
631
632
};

633
struct evl_factory evl_poll_factory = {
634
	.name	=	EVL_POLL_DEV,
635
636
	.fops	=	&poll_fops,
	.flags	=	EVL_FACTORY_SINGLE,
637
};