qdio_main.c 40.9 KB
Newer Older
Jan Glauber's avatar
Jan Glauber committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/*
 * linux/drivers/s390/cio/qdio_main.c
 *
 * Linux for s390 qdio support, buffer handling, qdio API and module support.
 *
 * Copyright 2000,2008 IBM Corp.
 * Author(s): Utz Bacher <utz.bacher@de.ibm.com>
 *	      Jan Glauber <jang@linux.vnet.ibm.com>
 * 2.6 cio integration by Cornelia Huck <cornelia.huck@de.ibm.com>
 */
#include <linux/module.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/timer.h>
#include <linux/delay.h>
16
#include <linux/gfp.h>
17
#include <linux/kernel_stat.h>
Arun Sharma's avatar
Arun Sharma committed
18
#include <linux/atomic.h>
Jan Glauber's avatar
Jan Glauber committed
19
20
21
22
23
24
25
26
27
28
29
30
31
32
#include <asm/debug.h>
#include <asm/qdio.h>

#include "cio.h"
#include "css.h"
#include "device.h"
#include "qdio.h"
#include "qdio_debug.h"

MODULE_AUTHOR("Utz Bacher <utz.bacher@de.ibm.com>,"\
	"Jan Glauber <jang@linux.vnet.ibm.com>");
MODULE_DESCRIPTION("QDIO base support");
MODULE_LICENSE("GPL");

33
34
35
static inline int do_siga_sync(unsigned long schid,
			       unsigned int out_mask, unsigned int in_mask,
			       unsigned int fc)
Jan Glauber's avatar
Jan Glauber committed
36
{
37
38
	register unsigned long __fc asm ("0") = fc;
	register unsigned long __schid asm ("1") = schid;
Jan Glauber's avatar
Jan Glauber committed
39
40
41
42
43
44
45
46
47
48
49
50
51
	register unsigned long out asm ("2") = out_mask;
	register unsigned long in asm ("3") = in_mask;
	int cc;

	asm volatile(
		"	siga	0\n"
		"	ipm	%0\n"
		"	srl	%0,28\n"
		: "=d" (cc)
		: "d" (__fc), "d" (__schid), "d" (out), "d" (in) : "cc");
	return cc;
}

52
53
static inline int do_siga_input(unsigned long schid, unsigned int mask,
				unsigned int fc)
Jan Glauber's avatar
Jan Glauber committed
54
{
55
56
	register unsigned long __fc asm ("0") = fc;
	register unsigned long __schid asm ("1") = schid;
Jan Glauber's avatar
Jan Glauber committed
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
	register unsigned long __mask asm ("2") = mask;
	int cc;

	asm volatile(
		"	siga	0\n"
		"	ipm	%0\n"
		"	srl	%0,28\n"
		: "=d" (cc)
		: "d" (__fc), "d" (__schid), "d" (__mask) : "cc", "memory");
	return cc;
}

/**
 * do_siga_output - perform SIGA-w/wt function
 * @schid: subchannel id or in case of QEBSM the subchannel token
 * @mask: which output queues to process
 * @bb: busy bit indicator, set only if SIGA-w/wt could not access a buffer
 * @fc: function code to perform
 *
 * Returns cc or QDIO_ERROR_SIGA_ACCESS_EXCEPTION.
 * Note: For IQDC unicast queues only the highest priority queue is processed.
 */
static inline int do_siga_output(unsigned long schid, unsigned long mask,
80
				 unsigned int *bb, unsigned int fc)
Jan Glauber's avatar
Jan Glauber committed
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
{
	register unsigned long __fc asm("0") = fc;
	register unsigned long __schid asm("1") = schid;
	register unsigned long __mask asm("2") = mask;
	int cc = QDIO_ERROR_SIGA_ACCESS_EXCEPTION;

	asm volatile(
		"	siga	0\n"
		"0:	ipm	%0\n"
		"	srl	%0,28\n"
		"1:\n"
		EX_TABLE(0b, 1b)
		: "+d" (cc), "+d" (__fc), "+d" (__schid), "+d" (__mask)
		: : "cc", "memory");
	*bb = ((unsigned int) __fc) >> 31;
	return cc;
}

static inline int qdio_check_ccq(struct qdio_q *q, unsigned int ccq)
{
	/* all done or next buffer state different */
	if (ccq == 0 || ccq == 32)
		return 0;
	/* not all buffers processed */
	if (ccq == 96 || ccq == 97)
		return 1;
	/* notify devices immediately */
108
	DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq);
Jan Glauber's avatar
Jan Glauber committed
109
110
111
112
113
114
115
116
117
	return -EIO;
}

/**
 * qdio_do_eqbs - extract buffer states for QEBSM
 * @q: queue to manipulate
 * @state: state of the extracted buffers
 * @start: buffer number to start at
 * @count: count of buffers to examine
118
 * @auto_ack: automatically acknowledge buffers
Jan Glauber's avatar
Jan Glauber committed
119
 *
Coly Li's avatar
Coly Li committed
120
 * Returns the number of successfully extracted equal buffer states.
Jan Glauber's avatar
Jan Glauber committed
121
122
123
 * Stops processing if a state is different from the last buffers state.
 */
static int qdio_do_eqbs(struct qdio_q *q, unsigned char *state,
124
			int start, int count, int auto_ack)
Jan Glauber's avatar
Jan Glauber committed
125
126
127
128
129
130
131
{
	unsigned int ccq = 0;
	int tmp_count = count, tmp_start = start;
	int nr = q->nr;
	int rc;

	BUG_ON(!q->irq_ptr->sch_token);
132
	qperf_inc(q, eqbs);
Jan Glauber's avatar
Jan Glauber committed
133
134
135
136

	if (!q->is_input_q)
		nr += q->irq_ptr->nr_input_qs;
again:
137
138
	ccq = do_eqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count,
		      auto_ack);
Jan Glauber's avatar
Jan Glauber committed
139
140
141
142
143
	rc = qdio_check_ccq(q, ccq);

	/* At least one buffer was processed, return and extract the remaining
	 * buffers later.
	 */
144
	if ((ccq == 96) && (count != tmp_count)) {
145
		qperf_inc(q, eqbs_partial);
Jan Glauber's avatar
Jan Glauber committed
146
		return (count - tmp_count);
147
	}
148

Jan Glauber's avatar
Jan Glauber committed
149
	if (rc == 1) {
150
		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS again:%2d", ccq);
Jan Glauber's avatar
Jan Glauber committed
151
152
153
154
		goto again;
	}

	if (rc < 0) {
155
156
		DBF_ERROR("%4x EQBS ERROR", SCH_NO(q));
		DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
Jan Glauber's avatar
Jan Glauber committed
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
		q->handler(q->irq_ptr->cdev,
			   QDIO_ERROR_ACTIVATE_CHECK_CONDITION,
			   0, -1, -1, q->irq_ptr->int_parm);
		return 0;
	}
	return count - tmp_count;
}

/**
 * qdio_do_sqbs - set buffer states for QEBSM
 * @q: queue to manipulate
 * @state: new state of the buffers
 * @start: first buffer number to change
 * @count: how many buffers to change
 *
 * Returns the number of successfully changed buffers.
 * Does retrying until the specified count of buffer states is set or an
 * error occurs.
 */
static int qdio_do_sqbs(struct qdio_q *q, unsigned char state, int start,
			int count)
{
	unsigned int ccq = 0;
	int tmp_count = count, tmp_start = start;
	int nr = q->nr;
	int rc;

184
185
186
	if (!count)
		return 0;

Jan Glauber's avatar
Jan Glauber committed
187
	BUG_ON(!q->irq_ptr->sch_token);
188
	qperf_inc(q, sqbs);
Jan Glauber's avatar
Jan Glauber committed
189
190
191
192
193
194
195

	if (!q->is_input_q)
		nr += q->irq_ptr->nr_input_qs;
again:
	ccq = do_sqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count);
	rc = qdio_check_ccq(q, ccq);
	if (rc == 1) {
196
		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "SQBS again:%2d", ccq);
197
		qperf_inc(q, sqbs_partial);
Jan Glauber's avatar
Jan Glauber committed
198
199
200
		goto again;
	}
	if (rc < 0) {
201
202
		DBF_ERROR("%4x SQBS ERROR", SCH_NO(q));
		DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
Jan Glauber's avatar
Jan Glauber committed
203
204
205
206
207
208
209
210
211
212
213
		q->handler(q->irq_ptr->cdev,
			   QDIO_ERROR_ACTIVATE_CHECK_CONDITION,
			   0, -1, -1, q->irq_ptr->int_parm);
		return 0;
	}
	WARN_ON(tmp_count);
	return count - tmp_count;
}

/* returns number of examined buffers and their common state in *state */
static inline int get_buf_states(struct qdio_q *q, unsigned int bufnr,
214
215
				 unsigned char *state, unsigned int count,
				 int auto_ack)
Jan Glauber's avatar
Jan Glauber committed
216
217
218
219
220
221
222
223
{
	unsigned char __state = 0;
	int i;

	BUG_ON(bufnr > QDIO_MAX_BUFFERS_MASK);
	BUG_ON(count > QDIO_MAX_BUFFERS_PER_Q);

	if (is_qebsm(q))
224
		return qdio_do_eqbs(q, state, bufnr, count, auto_ack);
Jan Glauber's avatar
Jan Glauber committed
225
226
227
228
229
230
231
232
233
234
235
236

	for (i = 0; i < count; i++) {
		if (!__state)
			__state = q->slsb.val[bufnr];
		else if (q->slsb.val[bufnr] != __state)
			break;
		bufnr = next_buf(bufnr);
	}
	*state = __state;
	return i;
}

237
238
static inline int get_buf_state(struct qdio_q *q, unsigned int bufnr,
				unsigned char *state, int auto_ack)
Jan Glauber's avatar
Jan Glauber committed
239
{
240
	return get_buf_states(q, bufnr, state, 1, auto_ack);
Jan Glauber's avatar
Jan Glauber committed
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
}

/* wrap-around safe setting of slsb states, returns number of changed buffers */
static inline int set_buf_states(struct qdio_q *q, int bufnr,
				 unsigned char state, int count)
{
	int i;

	BUG_ON(bufnr > QDIO_MAX_BUFFERS_MASK);
	BUG_ON(count > QDIO_MAX_BUFFERS_PER_Q);

	if (is_qebsm(q))
		return qdio_do_sqbs(q, state, bufnr, count);

	for (i = 0; i < count; i++) {
		xchg(&q->slsb.val[bufnr], state);
		bufnr = next_buf(bufnr);
	}
	return count;
}

static inline int set_buf_state(struct qdio_q *q, int bufnr,
				unsigned char state)
{
	return set_buf_states(q, bufnr, state, 1);
}

/* set slsb states to initial state */
void qdio_init_buf_states(struct qdio_irq *irq_ptr)
{
	struct qdio_q *q;
	int i;

	for_each_input_queue(irq_ptr, q, i)
		set_buf_states(q, 0, SLSB_P_INPUT_NOT_INIT,
			       QDIO_MAX_BUFFERS_PER_Q);
	for_each_output_queue(irq_ptr, q, i)
		set_buf_states(q, 0, SLSB_P_OUTPUT_NOT_INIT,
			       QDIO_MAX_BUFFERS_PER_Q);
}

282
static inline int qdio_siga_sync(struct qdio_q *q, unsigned int output,
Jan Glauber's avatar
Jan Glauber committed
283
284
			  unsigned int input)
{
285
286
	unsigned long schid = *((u32 *) &q->irq_ptr->schid);
	unsigned int fc = QDIO_SIGA_SYNC;
Jan Glauber's avatar
Jan Glauber committed
287
288
	int cc;

289
	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-s:%1d", q->nr);
290
	qperf_inc(q, siga_sync);
Jan Glauber's avatar
Jan Glauber committed
291

292
293
294
295
296
297
	if (is_qebsm(q)) {
		schid = q->irq_ptr->sch_token;
		fc |= QDIO_SIGA_QEBSM_FLAG;
	}

	cc = do_siga_sync(schid, output, input, fc);
298
	if (unlikely(cc))
299
		DBF_ERROR("%4x SIGA-S:%2d", SCH_NO(q), cc);
Jan Glauber's avatar
Jan Glauber committed
300
301
302
	return cc;
}

303
static inline int qdio_siga_sync_q(struct qdio_q *q)
Jan Glauber's avatar
Jan Glauber committed
304
305
306
307
308
309
310
{
	if (q->is_input_q)
		return qdio_siga_sync(q, 0, q->mask);
	else
		return qdio_siga_sync(q, q->mask, 0);
}

311
static int qdio_siga_output(struct qdio_q *q, unsigned int *busy_bit)
Jan Glauber's avatar
Jan Glauber committed
312
{
313
314
	unsigned long schid = *((u32 *) &q->irq_ptr->schid);
	unsigned int fc = QDIO_SIGA_WRITE;
315
	u64 start_time = 0;
316
	int retries = 0, cc;
Jan Glauber's avatar
Jan Glauber committed
317

318
	if (is_qebsm(q)) {
Jan Glauber's avatar
Jan Glauber committed
319
		schid = q->irq_ptr->sch_token;
320
		fc |= QDIO_SIGA_QEBSM_FLAG;
Jan Glauber's avatar
Jan Glauber committed
321
322
	}
again:
323
324
325
	cc = do_siga_output(schid, q->mask, busy_bit, fc);

	/* hipersocket busy condition */
326
	if (unlikely(*busy_bit)) {
327
		WARN_ON(queue_type(q) != QDIO_IQDIO_QFMT || cc != 2);
328
		retries++;
329

330
		if (!start_time) {
331
			start_time = get_clock();
332
333
			goto again;
		}
334
		if ((get_clock() - start_time) < QDIO_BUSY_BIT_PATIENCE)
Jan Glauber's avatar
Jan Glauber committed
335
336
			goto again;
	}
337
338
339
340
341
	if (retries) {
		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr,
			      "%4x cc2 BB1:%1d", SCH_NO(q), q->nr);
		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "count:%u", retries);
	}
Jan Glauber's avatar
Jan Glauber committed
342
343
344
345
346
	return cc;
}

static inline int qdio_siga_input(struct qdio_q *q)
{
347
348
	unsigned long schid = *((u32 *) &q->irq_ptr->schid);
	unsigned int fc = QDIO_SIGA_READ;
Jan Glauber's avatar
Jan Glauber committed
349
350
	int cc;

351
	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-r:%1d", q->nr);
352
	qperf_inc(q, siga_read);
Jan Glauber's avatar
Jan Glauber committed
353

354
355
356
357
358
359
	if (is_qebsm(q)) {
		schid = q->irq_ptr->sch_token;
		fc |= QDIO_SIGA_QEBSM_FLAG;
	}

	cc = do_siga_input(schid, q->mask, fc);
360
	if (unlikely(cc))
361
		DBF_ERROR("%4x SIGA-R:%2d", SCH_NO(q), cc);
Jan Glauber's avatar
Jan Glauber committed
362
363
364
	return cc;
}

Jan Glauber's avatar
Jan Glauber committed
365
366
367
368
#define qdio_siga_sync_out(q) qdio_siga_sync(q, ~0U, 0)
#define qdio_siga_sync_all(q) qdio_siga_sync(q, ~0U, ~0U)

static inline void qdio_sync_queues(struct qdio_q *q)
Jan Glauber's avatar
Jan Glauber committed
369
{
Jan Glauber's avatar
Jan Glauber committed
370
371
372
373
	/* PCI capable outbound queues will also be scanned so sync them too */
	if (pci_out_supported(q))
		qdio_siga_sync_all(q);
	else
Jan Glauber's avatar
Jan Glauber committed
374
375
376
		qdio_siga_sync_q(q);
}

377
378
379
int debug_get_buf_state(struct qdio_q *q, unsigned int bufnr,
			unsigned char *state)
{
Jan Glauber's avatar
Jan Glauber committed
380
381
	if (need_siga_sync(q))
		qdio_siga_sync_q(q);
382
383
384
385
	return get_buf_states(q, bufnr, state, 1, 0);
}

static inline void qdio_stop_polling(struct qdio_q *q)
Jan Glauber's avatar
Jan Glauber committed
386
{
387
	if (!q->u.in.polling)
Jan Glauber's avatar
Jan Glauber committed
388
		return;
389

Jan Glauber's avatar
Jan Glauber committed
390
	q->u.in.polling = 0;
391
	qperf_inc(q, stop_polling);
Jan Glauber's avatar
Jan Glauber committed
392
393

	/* show the card that we are not polling anymore */
394
	if (is_qebsm(q)) {
395
		set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT,
396
397
398
			       q->u.in.ack_count);
		q->u.in.ack_count = 0;
	} else
399
		set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT);
Jan Glauber's avatar
Jan Glauber committed
400
401
}

402
403
404
405
406
407
408
409
410
411
412
413
414
415
static inline void account_sbals(struct qdio_q *q, int count)
{
	int pos = 0;

	q->q_stats.nr_sbal_total += count;
	if (count == QDIO_MAX_BUFFERS_MASK) {
		q->q_stats.nr_sbals[7]++;
		return;
	}
	while (count >>= 1)
		pos++;
	q->q_stats.nr_sbals[pos]++;
}

416
static void process_buffer_error(struct qdio_q *q, int count)
Jan Glauber's avatar
Jan Glauber committed
417
{
418
419
420
	unsigned char state = (q->is_input_q) ? SLSB_P_INPUT_NOT_INIT :
					SLSB_P_OUTPUT_NOT_INIT;

421
	q->qdio_error |= QDIO_ERROR_SLSB_STATE;
422
423
424

	/* special handling for no target buffer empty */
	if ((!q->is_input_q &&
425
	    (q->sbal[q->first_to_check]->element[15].sflags) == 0x10)) {
426
		qperf_inc(q, target_full);
427
		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x",
428
429
430
431
			      q->first_to_check);
		return;
	}

432
433
	DBF_ERROR("%4x BUF ERROR", SCH_NO(q));
	DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr);
434
	DBF_ERROR("FTC:%3d C:%3d", q->first_to_check, count);
435
	DBF_ERROR("F14:%2x F15:%2x",
436
437
		  q->sbal[q->first_to_check]->element[14].sflags,
		  q->sbal[q->first_to_check]->element[15].sflags);
438
439
440
441
442
443

	/*
	 * Interrupts may be avoided as long as the error is present
	 * so change the buffer state immediately to avoid starvation.
	 */
	set_buf_states(q, q->first_to_check, state, count);
444
}
Jan Glauber's avatar
Jan Glauber committed
445

446
447
448
449
static inline void inbound_primed(struct qdio_q *q, int count)
{
	int new;

450
	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim: %02x", count);
451
452
453
454
455
456

	/* for QEBSM the ACK was already set by EQBS */
	if (is_qebsm(q)) {
		if (!q->u.in.polling) {
			q->u.in.polling = 1;
			q->u.in.ack_count = count;
457
			q->u.in.ack_start = q->first_to_check;
458
459
460
461
			return;
		}

		/* delete the previous ACK's */
462
		set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT,
463
464
			       q->u.in.ack_count);
		q->u.in.ack_count = count;
465
		q->u.in.ack_start = q->first_to_check;
466
467
468
469
470
471
472
473
474
475
476
		return;
	}

	/*
	 * ACK the newest buffer. The ACK will be removed in qdio_stop_polling
	 * or by the next inbound run.
	 */
	new = add_buf(q->first_to_check, count - 1);
	if (q->u.in.polling) {
		/* reset the previous ACK but first set the new one */
		set_buf_state(q, new, SLSB_P_INPUT_ACK);
477
		set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT);
478
	} else {
479
		q->u.in.polling = 1;
480
		set_buf_state(q, new, SLSB_P_INPUT_ACK);
481
482
	}

483
	q->u.in.ack_start = new;
484
485
486
	count--;
	if (!count)
		return;
487
488
	/* need to change ALL buffers to get more interrupts */
	set_buf_states(q, q->first_to_check, SLSB_P_INPUT_NOT_INIT, count);
Jan Glauber's avatar
Jan Glauber committed
489
490
491
492
493
}

static int get_inbound_buffer_frontier(struct qdio_q *q)
{
	int count, stop;
494
	unsigned char state = 0;
Jan Glauber's avatar
Jan Glauber committed
495
496
497
498
499
500
501
502
503
504
505

	/*
	 * Don't check 128 buffers, as otherwise qdio_inbound_q_moved
	 * would return 0.
	 */
	count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK);
	stop = add_buf(q->first_to_check, count);

	if (q->first_to_check == stop)
		goto out;

506
507
508
509
	/*
	 * No siga sync here, as a PCI or we after a thin interrupt
	 * already sync'ed the queues.
	 */
510
	count = get_buf_states(q, q->first_to_check, &state, count, 1);
Jan Glauber's avatar
Jan Glauber committed
511
512
513
514
515
	if (!count)
		goto out;

	switch (state) {
	case SLSB_P_INPUT_PRIMED:
516
		inbound_primed(q, count);
Jan Glauber's avatar
Jan Glauber committed
517
		q->first_to_check = add_buf(q->first_to_check, count);
518
		if (atomic_sub(count, &q->nr_buf_used) == 0)
519
			qperf_inc(q, inbound_queue_full);
520
521
		if (q->irq_ptr->perf_stat_enabled)
			account_sbals(q, count);
522
		break;
Jan Glauber's avatar
Jan Glauber committed
523
	case SLSB_P_INPUT_ERROR:
524
		process_buffer_error(q, count);
Jan Glauber's avatar
Jan Glauber committed
525
526
		q->first_to_check = add_buf(q->first_to_check, count);
		atomic_sub(count, &q->nr_buf_used);
527
528
		if (q->irq_ptr->perf_stat_enabled)
			account_sbals_error(q, count);
Jan Glauber's avatar
Jan Glauber committed
529
530
531
532
		break;
	case SLSB_CU_INPUT_EMPTY:
	case SLSB_P_INPUT_NOT_INIT:
	case SLSB_P_INPUT_ACK:
533
534
		if (q->irq_ptr->perf_stat_enabled)
			q->q_stats.nr_sbal_nop++;
535
		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop");
Jan Glauber's avatar
Jan Glauber committed
536
537
538
539
540
541
542
543
		break;
	default:
		BUG();
	}
out:
	return q->first_to_check;
}

544
static int qdio_inbound_q_moved(struct qdio_q *q)
Jan Glauber's avatar
Jan Glauber committed
545
546
547
548
549
{
	int bufnr;

	bufnr = get_inbound_buffer_frontier(q);

550
551
	if ((bufnr != q->last_move) || q->qdio_error) {
		q->last_move = bufnr;
552
		if (!is_thinint_irq(q->irq_ptr) && MACHINE_IS_LPAR)
553
			q->u.in.timestamp = get_clock();
Jan Glauber's avatar
Jan Glauber committed
554
555
556
557
558
		return 1;
	} else
		return 0;
}

559
static inline int qdio_inbound_q_done(struct qdio_q *q)
Jan Glauber's avatar
Jan Glauber committed
560
{
561
	unsigned char state = 0;
Jan Glauber's avatar
Jan Glauber committed
562
563
564
565

	if (!atomic_read(&q->nr_buf_used))
		return 1;

Jan Glauber's avatar
Jan Glauber committed
566
567
	if (need_siga_sync(q))
		qdio_siga_sync_q(q);
568
	get_buf_state(q, q->first_to_check, &state, 0);
569

570
	if (state == SLSB_P_INPUT_PRIMED || state == SLSB_P_INPUT_ERROR)
571
		/* more work coming */
Jan Glauber's avatar
Jan Glauber committed
572
573
		return 0;

574
575
576
577
578
	if (is_thinint_irq(q->irq_ptr))
		return 1;

	/* don't poll under z/VM */
	if (MACHINE_IS_VM)
Jan Glauber's avatar
Jan Glauber committed
579
580
581
582
583
584
		return 1;

	/*
	 * At this point we know, that inbound first_to_check
	 * has (probably) not moved (see qdio_inbound_processing).
	 */
585
	if (get_clock() > q->u.in.timestamp + QDIO_INPUT_THRESHOLD) {
586
		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in done:%02x",
587
			      q->first_to_check);
Jan Glauber's avatar
Jan Glauber committed
588
		return 1;
589
	} else
590
591
592
593
		return 0;
}

static void qdio_kick_handler(struct qdio_q *q)
Jan Glauber's avatar
Jan Glauber committed
594
{
595
596
597
	int start = q->first_to_kick;
	int end = q->first_to_check;
	int count;
Jan Glauber's avatar
Jan Glauber committed
598
599
600
601

	if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
		return;

602
603
604
	count = sub_buf(end, start);

	if (q->is_input_q) {
605
		qperf_inc(q, inbound_handler);
606
		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "kih s:%02x c:%02x", start, count);
607
	} else {
608
		qperf_inc(q, outbound_handler);
609
610
		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "koh: s:%02x c:%02x",
			      start, count);
611
	}
612
613
614

	q->handler(q->irq_ptr->cdev, q->qdio_error, q->nr, start, count,
		   q->irq_ptr->int_parm);
Jan Glauber's avatar
Jan Glauber committed
615
616

	/* for the next time */
617
	q->first_to_kick = end;
Jan Glauber's avatar
Jan Glauber committed
618
619
620
621
622
	q->qdio_error = 0;
}

static void __qdio_inbound_processing(struct qdio_q *q)
{
623
	qperf_inc(q, tasklet_inbound);
624

Jan Glauber's avatar
Jan Glauber committed
625
626
627
	if (!qdio_inbound_q_moved(q))
		return;

628
	qdio_kick_handler(q);
Jan Glauber's avatar
Jan Glauber committed
629

630
	if (!qdio_inbound_q_done(q)) {
Jan Glauber's avatar
Jan Glauber committed
631
		/* means poll time is not yet over */
632
		qperf_inc(q, tasklet_inbound_resched);
633
634
635
636
		if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED)) {
			tasklet_schedule(&q->tasklet);
			return;
		}
637
	}
Jan Glauber's avatar
Jan Glauber committed
638
639
640
641
642
643

	qdio_stop_polling(q);
	/*
	 * We need to check again to not lose initiative after
	 * resetting the ACK state.
	 */
644
645
	if (!qdio_inbound_q_done(q)) {
		qperf_inc(q, tasklet_inbound_resched2);
646
647
		if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED))
			tasklet_schedule(&q->tasklet);
648
	}
Jan Glauber's avatar
Jan Glauber committed
649
650
651
652
653
654
655
656
657
658
659
}

void qdio_inbound_processing(unsigned long data)
{
	struct qdio_q *q = (struct qdio_q *)data;
	__qdio_inbound_processing(q);
}

static int get_outbound_buffer_frontier(struct qdio_q *q)
{
	int count, stop;
660
	unsigned char state = 0;
Jan Glauber's avatar
Jan Glauber committed
661

Jan Glauber's avatar
Jan Glauber committed
662
663
664
665
666
667
	if (need_siga_sync(q))
		if (((queue_type(q) != QDIO_IQDIO_QFMT) &&
		    !pci_out_supported(q)) ||
		    (queue_type(q) == QDIO_IQDIO_QFMT &&
		    multicast_outbound(q)))
			qdio_siga_sync_q(q);
Jan Glauber's avatar
Jan Glauber committed
668
669
670
671
672
673
674
675
676
677
678

	/*
	 * Don't check 128 buffers, as otherwise qdio_inbound_q_moved
	 * would return 0.
	 */
	count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK);
	stop = add_buf(q->first_to_check, count);

	if (q->first_to_check == stop)
		return q->first_to_check;

679
	count = get_buf_states(q, q->first_to_check, &state, count, 0);
Jan Glauber's avatar
Jan Glauber committed
680
681
682
683
684
685
	if (!count)
		return q->first_to_check;

	switch (state) {
	case SLSB_P_OUTPUT_EMPTY:
		/* the adapter got it */
686
		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out empty:%1d %02x", q->nr, count);
Jan Glauber's avatar
Jan Glauber committed
687
688
689

		atomic_sub(count, &q->nr_buf_used);
		q->first_to_check = add_buf(q->first_to_check, count);
690
691
		if (q->irq_ptr->perf_stat_enabled)
			account_sbals(q, count);
692
		break;
Jan Glauber's avatar
Jan Glauber committed
693
	case SLSB_P_OUTPUT_ERROR:
694
		process_buffer_error(q, count);
Jan Glauber's avatar
Jan Glauber committed
695
696
		q->first_to_check = add_buf(q->first_to_check, count);
		atomic_sub(count, &q->nr_buf_used);
697
698
		if (q->irq_ptr->perf_stat_enabled)
			account_sbals_error(q, count);
Jan Glauber's avatar
Jan Glauber committed
699
700
701
		break;
	case SLSB_CU_OUTPUT_PRIMED:
		/* the adapter has not fetched the output yet */
702
703
		if (q->irq_ptr->perf_stat_enabled)
			q->q_stats.nr_sbal_nop++;
704
		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d", q->nr);
Jan Glauber's avatar
Jan Glauber committed
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
		break;
	case SLSB_P_OUTPUT_NOT_INIT:
	case SLSB_P_OUTPUT_HALTED:
		break;
	default:
		BUG();
	}
	return q->first_to_check;
}

/* all buffers processed? */
static inline int qdio_outbound_q_done(struct qdio_q *q)
{
	return atomic_read(&q->nr_buf_used) == 0;
}

static inline int qdio_outbound_q_moved(struct qdio_q *q)
{
	int bufnr;

	bufnr = get_outbound_buffer_frontier(q);

727
728
	if ((bufnr != q->last_move) || q->qdio_error) {
		q->last_move = bufnr;
729
		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out moved:%1d", q->nr);
Jan Glauber's avatar
Jan Glauber committed
730
731
732
733
734
		return 1;
	} else
		return 0;
}

735
static int qdio_kick_outbound_q(struct qdio_q *q)
Jan Glauber's avatar
Jan Glauber committed
736
{
737
	int retries = 0, cc;
738
	unsigned int busy_bit;
Jan Glauber's avatar
Jan Glauber committed
739
740

	if (!need_siga_out(q))
741
		return 0;
Jan Glauber's avatar
Jan Glauber committed
742

743
	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w:%1d", q->nr);
744
retry:
745
	qperf_inc(q, siga_write);
746
747
748

	cc = qdio_siga_output(q, &busy_bit);
	switch (cc) {
Jan Glauber's avatar
Jan Glauber committed
749
750
	case 0:
		break;
751
752
	case 2:
		if (busy_bit) {
753
754
755
756
757
			while (++retries < QDIO_BUSY_BIT_RETRIES) {
				mdelay(QDIO_BUSY_BIT_RETRY_DELAY);
				goto retry;
			}
			DBF_ERROR("%4x cc2 BBC:%1d", SCH_NO(q), q->nr);
758
759
760
			cc |= QDIO_ERROR_SIGA_BUSY;
		} else
			DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w cc2:%1d", q->nr);
761
762
763
764
765
		break;
	case 1:
	case 3:
		DBF_ERROR("%4x SIGA-W:%1d", SCH_NO(q), cc);
		break;
Jan Glauber's avatar
Jan Glauber committed
766
	}
767
768
769
770
	if (retries) {
		DBF_ERROR("%4x cc2 BB2:%1d", SCH_NO(q), q->nr);
		DBF_ERROR("count:%u", retries);
	}
771
	return cc;
Jan Glauber's avatar
Jan Glauber committed
772
773
774
775
}

static void __qdio_outbound_processing(struct qdio_q *q)
{
776
	qperf_inc(q, tasklet_outbound);
Jan Glauber's avatar
Jan Glauber committed
777
778
779
	BUG_ON(atomic_read(&q->nr_buf_used) < 0);

	if (qdio_outbound_q_moved(q))
780
		qdio_kick_handler(q);
Jan Glauber's avatar
Jan Glauber committed
781

782
	if (queue_type(q) == QDIO_ZFCP_QFMT)
Jan Glauber's avatar
Jan Glauber committed
783
		if (!pci_out_supported(q) && !qdio_outbound_q_done(q))
784
			goto sched;
Jan Glauber's avatar
Jan Glauber committed
785
786
787
788
789

	/* bail out for HiperSockets unicast queues */
	if (queue_type(q) == QDIO_IQDIO_QFMT && !multicast_outbound(q))
		return;

790
	if ((queue_type(q) == QDIO_IQDIO_QFMT) &&
791
792
	    (atomic_read(&q->nr_buf_used)) > QDIO_IQDIO_POLL_LVL)
		goto sched;
793

Jan Glauber's avatar
Jan Glauber committed
794
795
796
797
798
799
800
801
802
803
	if (q->u.out.pci_out_enabled)
		return;

	/*
	 * Now we know that queue type is either qeth without pci enabled
	 * or HiperSockets multicast. Make sure buffer switch from PRIMED to
	 * EMPTY is noticed and outbound_handler is called after some time.
	 */
	if (qdio_outbound_q_done(q))
		del_timer(&q->u.out.timer);
804
805
	else
		if (!timer_pending(&q->u.out.timer))
Jan Glauber's avatar
Jan Glauber committed
806
			mod_timer(&q->u.out.timer, jiffies + 10 * HZ);
807
808
809
810
811
812
	return;

sched:
	if (unlikely(q->irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
		return;
	tasklet_schedule(&q->tasklet);
Jan Glauber's avatar
Jan Glauber committed
813
814
815
816
817
818
819
820
821
822
823
824
}

/* outbound tasklet */
void qdio_outbound_processing(unsigned long data)
{
	struct qdio_q *q = (struct qdio_q *)data;
	__qdio_outbound_processing(q);
}

void qdio_outbound_timer(unsigned long data)
{
	struct qdio_q *q = (struct qdio_q *)data;
825
826
827

	if (unlikely(q->irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
		return;
Jan Glauber's avatar
Jan Glauber committed
828
829
830
	tasklet_schedule(&q->tasklet);
}

831
static inline void qdio_check_outbound_after_thinint(struct qdio_q *q)
Jan Glauber's avatar
Jan Glauber committed
832
833
834
835
836
837
838
839
840
841
842
843
{
	struct qdio_q *out;
	int i;

	if (!pci_out_supported(q))
		return;

	for_each_output_queue(q->irq_ptr, out, i)
		if (!qdio_outbound_q_done(out))
			tasklet_schedule(&out->tasklet);
}

844
845
static void __tiqdio_inbound_processing(struct qdio_q *q)
{
846
	qperf_inc(q, tasklet_inbound);
Jan Glauber's avatar
Jan Glauber committed
847
848
	if (need_siga_sync(q) && need_siga_sync_after_ai(q))
		qdio_sync_queues(q);
849
850
851
852
853
854
855
856
857
858
859
860

	/*
	 * The interrupt could be caused by a PCI request. Check the
	 * PCI capable outbound queues.
	 */
	qdio_check_outbound_after_thinint(q);

	if (!qdio_inbound_q_moved(q))
		return;

	qdio_kick_handler(q);

861
	if (!qdio_inbound_q_done(q)) {
862
		qperf_inc(q, tasklet_inbound_resched);
863
		if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED)) {
864
			tasklet_schedule(&q->tasklet);
865
866
			return;
		}
867
868
869
870
871
872
873
	}

	qdio_stop_polling(q);
	/*
	 * We need to check again to not lose initiative after
	 * resetting the ACK state.
	 */
874
	if (!qdio_inbound_q_done(q)) {
875
		qperf_inc(q, tasklet_inbound_resched2);
876
877
878
879
880
881
882
883
884
885
886
		if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED))
			tasklet_schedule(&q->tasklet);
	}
}

void tiqdio_inbound_processing(unsigned long data)
{
	struct qdio_q *q = (struct qdio_q *)data;
	__tiqdio_inbound_processing(q);
}

Jan Glauber's avatar
Jan Glauber committed
887
888
889
static inline void qdio_set_state(struct qdio_irq *irq_ptr,
				  enum qdio_irq_states state)
{
890
	DBF_DEV_EVENT(DBF_INFO, irq_ptr, "newstate: %1d", state);
Jan Glauber's avatar
Jan Glauber committed
891
892
893
894
895

	irq_ptr->state = state;
	mb();
}

896
static void qdio_irq_check_sense(struct qdio_irq *irq_ptr, struct irb *irb)
Jan Glauber's avatar
Jan Glauber committed
897
898
{
	if (irb->esw.esw0.erw.cons) {
899
900
901
		DBF_ERROR("%4x sense:", irq_ptr->schid.sch_no);
		DBF_ERROR_HEX(irb, 64);
		DBF_ERROR_HEX(irb->ecw, 64);
Jan Glauber's avatar
Jan Glauber committed
902
903
904
905
906
907
908
909
910
	}
}

/* PCI interrupt handler */
static void qdio_int_handler_pci(struct qdio_irq *irq_ptr)
{
	int i;
	struct qdio_q *q;

911
912
913
	if (unlikely(irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
		return;

914
915
916
917
918
919
920
921
922
923
924
925
926
	for_each_input_queue(irq_ptr, q, i) {
		if (q->u.in.queue_start_poll) {
			/* skip if polling is enabled or already in work */
			if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
				     &q->u.in.queue_irq_state)) {
				qperf_inc(q, int_discarded);
				continue;
			}
			q->u.in.queue_start_poll(q->irq_ptr->cdev, q->nr,
						 q->irq_ptr->int_parm);
		} else
			tasklet_schedule(&q->tasklet);
	}
Jan Glauber's avatar
Jan Glauber committed
927

Jan Glauber's avatar
Jan Glauber committed
928
	if (!pci_out_supported(q))
Jan Glauber's avatar
Jan Glauber committed
929
930
931
932
933
		return;

	for_each_output_queue(irq_ptr, q, i) {
		if (qdio_outbound_q_done(q))
			continue;
Jan Glauber's avatar
Jan Glauber committed
934
		if (need_siga_sync(q) && need_siga_sync_out_after_pci(q))
Jan Glauber's avatar
Jan Glauber committed
935
936
937
938
939
940
941
942
943
944
945
			qdio_siga_sync_q(q);
		tasklet_schedule(&q->tasklet);
	}
}

static void qdio_handle_activate_check(struct ccw_device *cdev,
				unsigned long intparm, int cstat, int dstat)
{
	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
	struct qdio_q *q;

946
947
948
	DBF_ERROR("%4x ACT CHECK", irq_ptr->schid.sch_no);
	DBF_ERROR("intp :%lx", intparm);
	DBF_ERROR("ds: %2x cs:%2x", dstat, cstat);
Jan Glauber's avatar
Jan Glauber committed
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963

	if (irq_ptr->nr_input_qs) {
		q = irq_ptr->input_qs[0];
	} else if (irq_ptr->nr_output_qs) {
		q = irq_ptr->output_qs[0];
	} else {
		dump_stack();
		goto no_handler;
	}
	q->handler(q->irq_ptr->cdev, QDIO_ERROR_ACTIVATE_CHECK_CONDITION,
		   0, -1, -1, irq_ptr->int_parm);
no_handler:
	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
}

964
965
static void qdio_establish_handle_irq(struct ccw_device *cdev, int cstat,
				      int dstat)
Jan Glauber's avatar
Jan Glauber committed
966
967
968
{
	struct qdio_irq *irq_ptr = cdev->private->qdio_data;

969
	DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qest irq");
Jan Glauber's avatar
Jan Glauber committed
970

971
	if (cstat)
Jan Glauber's avatar
Jan Glauber committed
972
		goto error;
973
	if (dstat & ~(DEV_STAT_DEV_END | DEV_STAT_CHN_END))
Jan Glauber's avatar
Jan Glauber committed
974
		goto error;
975
976
977
978
979
	if (!(dstat & DEV_STAT_DEV_END))
		goto error;
	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ESTABLISHED);
	return;

Jan Glauber's avatar
Jan Glauber committed
980
error:
981
982
	DBF_ERROR("%4x EQ:error", irq_ptr->schid.sch_no);
	DBF_ERROR("ds: %2x cs:%2x", dstat, cstat);
Jan Glauber's avatar
Jan Glauber committed
983
984
985
986
987
988
989
990
991
992
993
	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
}

/* qdio interrupt handler */
void qdio_int_handler(struct ccw_device *cdev, unsigned long intparm,
		      struct irb *irb)
{
	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
	int cstat, dstat;

	if (!intparm || !irq_ptr) {
994
		DBF_ERROR("qint:%4x", cdev->private->schid.sch_no);
Jan Glauber's avatar
Jan Glauber committed
995
996
997
		return;
	}

998
	kstat_cpu(smp_processor_id()).irqs[IOINT_QDI]++;
999
1000
	if (irq_ptr->perf_stat_enabled)
		irq_ptr->perf_stat.qdio_int++;