dasd.c 66 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
/*
 * File...........: linux/drivers/s390/block/dasd.c
 * Author(s)......: Holger Smolinski <Holger.Smolinski@de.ibm.com>
 *		    Horst Hummel <Horst.Hummel@de.ibm.com>
 *		    Carsten Otte <Cotte@de.ibm.com>
 *		    Martin Schwidefsky <schwidefsky@de.ibm.com>
 * Bugreports.to..: <Linux390@de.ibm.com>
 * (C) IBM Corporation, IBM Deutschland Entwicklung GmbH, 1999-2001
 *
 */

#include <linux/kmod.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/ctype.h>
#include <linux/major.h>
#include <linux/slab.h>
#include <linux/buffer_head.h>
19
#include <linux/hdreg.h>
Linus Torvalds's avatar
Linus Torvalds committed
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

#include <asm/ccwdev.h>
#include <asm/ebcdic.h>
#include <asm/idals.h>
#include <asm/todclk.h>

/* This is ugly... */
#define PRINTK_HEADER "dasd:"

#include "dasd_int.h"
/*
 * SECTION: Constant definitions to be used within this file
 */
#define DASD_CHANQ_MAX_SIZE 4

/*
 * SECTION: exported variables of dasd.c
 */
debug_info_t *dasd_debug_area;
struct dasd_discipline *dasd_diag_discipline_pointer;
40
void dasd_int_handler(struct ccw_device *, unsigned long, struct irb *);
Linus Torvalds's avatar
Linus Torvalds committed
41
42
43
44
45
46
47
48
49
50

MODULE_AUTHOR("Holger Smolinski <Holger.Smolinski@de.ibm.com>");
MODULE_DESCRIPTION("Linux on S/390 DASD device driver,"
		   " Copyright 2000 IBM Corporation");
MODULE_SUPPORTED_DEVICE("dasd");
MODULE_LICENSE("GPL");

/*
 * SECTION: prototypes for static functions of dasd.c
 */
51
52
53
54
55
56
57
static int  dasd_alloc_queue(struct dasd_block *);
static void dasd_setup_queue(struct dasd_block *);
static void dasd_free_queue(struct dasd_block *);
static void dasd_flush_request_queue(struct dasd_block *);
static int dasd_flush_block_queue(struct dasd_block *);
static void dasd_device_tasklet(struct dasd_device *);
static void dasd_block_tasklet(struct dasd_block *);
58
static void do_kick_device(struct work_struct *);
59
static void dasd_return_cqr_cb(struct dasd_ccw_req *, void *);
Linus Torvalds's avatar
Linus Torvalds committed
60
61
62
63
64

/*
 * SECTION: Operations on the device structure.
 */
static wait_queue_head_t dasd_init_waitq;
65
static wait_queue_head_t dasd_flush_wq;
66
static wait_queue_head_t generic_waitq;
Linus Torvalds's avatar
Linus Torvalds committed
67
68
69
70

/*
 * Allocate memory for a new device structure.
 */
71
struct dasd_device *dasd_alloc_device(void)
Linus Torvalds's avatar
Linus Torvalds committed
72
73
74
{
	struct dasd_device *device;

75
76
	device = kzalloc(sizeof(struct dasd_device), GFP_ATOMIC);
	if (!device)
Linus Torvalds's avatar
Linus Torvalds committed
77
78
79
80
		return ERR_PTR(-ENOMEM);

	/* Get two pages for normal block device operations. */
	device->ccw_mem = (void *) __get_free_pages(GFP_ATOMIC | GFP_DMA, 1);
81
	if (!device->ccw_mem) {
Linus Torvalds's avatar
Linus Torvalds committed
82
83
84
85
86
		kfree(device);
		return ERR_PTR(-ENOMEM);
	}
	/* Get one page for error recovery. */
	device->erp_mem = (void *) get_zeroed_page(GFP_ATOMIC | GFP_DMA);
87
	if (!device->erp_mem) {
Linus Torvalds's avatar
Linus Torvalds committed
88
89
90
91
92
93
94
95
		free_pages((unsigned long) device->ccw_mem, 1);
		kfree(device);
		return ERR_PTR(-ENOMEM);
	}

	dasd_init_chunklist(&device->ccw_chunks, device->ccw_mem, PAGE_SIZE*2);
	dasd_init_chunklist(&device->erp_chunks, device->erp_mem, PAGE_SIZE);
	spin_lock_init(&device->mem_lock);
96
	atomic_set(&device->tasklet_scheduled, 0);
97
	tasklet_init(&device->tasklet,
98
		     (void (*)(unsigned long)) dasd_device_tasklet,
Linus Torvalds's avatar
Linus Torvalds committed
99
100
101
		     (unsigned long) device);
	INIT_LIST_HEAD(&device->ccw_queue);
	init_timer(&device->timer);
102
	INIT_WORK(&device->kick_work, do_kick_device);
Linus Torvalds's avatar
Linus Torvalds committed
103
104
105
106
107
108
109
110
111
	device->state = DASD_STATE_NEW;
	device->target = DASD_STATE_NEW;

	return device;
}

/*
 * Free memory of a device structure.
 */
112
void dasd_free_device(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
113
{
114
	kfree(device->private);
Linus Torvalds's avatar
Linus Torvalds committed
115
116
117
118
119
	free_page((unsigned long) device->erp_mem);
	free_pages((unsigned long) device->ccw_mem, 1);
	kfree(device);
}

120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
/*
 * Allocate memory for a new device structure.
 */
struct dasd_block *dasd_alloc_block(void)
{
	struct dasd_block *block;

	block = kzalloc(sizeof(*block), GFP_ATOMIC);
	if (!block)
		return ERR_PTR(-ENOMEM);
	/* open_count = 0 means device online but not in use */
	atomic_set(&block->open_count, -1);

	spin_lock_init(&block->request_queue_lock);
	atomic_set(&block->tasklet_scheduled, 0);
	tasklet_init(&block->tasklet,
		     (void (*)(unsigned long)) dasd_block_tasklet,
		     (unsigned long) block);
	INIT_LIST_HEAD(&block->ccw_queue);
	spin_lock_init(&block->queue_lock);
	init_timer(&block->timer);

	return block;
}

/*
 * Free memory of a device structure.
 */
void dasd_free_block(struct dasd_block *block)
{
	kfree(block);
}

Linus Torvalds's avatar
Linus Torvalds committed
153
154
155
/*
 * Make a new device known to the system.
 */
156
static int dasd_state_new_to_known(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
157
158
159
160
{
	int rc;

	/*
161
	 * As long as the device is not in state DASD_STATE_NEW we want to
Linus Torvalds's avatar
Linus Torvalds committed
162
163
164
165
	 * keep the reference count > 0.
	 */
	dasd_get_device(device);

166
167
168
169
170
171
	if (device->block) {
		rc = dasd_alloc_queue(device->block);
		if (rc) {
			dasd_put_device(device);
			return rc;
		}
Linus Torvalds's avatar
Linus Torvalds committed
172
173
174
175
176
177
178
179
	}
	device->state = DASD_STATE_KNOWN;
	return 0;
}

/*
 * Let the system forget about a device.
 */
180
static int dasd_state_known_to_new(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
181
{
182
183
	/* Disable extended error reporting for this device. */
	dasd_eer_disable(device);
Linus Torvalds's avatar
Linus Torvalds committed
184
	/* Forget the discipline information. */
185
186
187
	if (device->discipline) {
		if (device->discipline->uncheck_device)
			device->discipline->uncheck_device(device);
188
		module_put(device->discipline->owner);
189
	}
Linus Torvalds's avatar
Linus Torvalds committed
190
	device->discipline = NULL;
191
192
193
	if (device->base_discipline)
		module_put(device->base_discipline->owner);
	device->base_discipline = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
194
195
	device->state = DASD_STATE_NEW;

196
197
	if (device->block)
		dasd_free_queue(device->block);
Linus Torvalds's avatar
Linus Torvalds committed
198
199
200

	/* Give up reference we took in dasd_state_new_to_known. */
	dasd_put_device(device);
201
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
202
203
204
205
206
}

/*
 * Request the irq line for the device.
 */
207
static int dasd_state_known_to_basic(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
208
209
210
211
{
	int rc;

	/* Allocate and register gendisk structure. */
212
213
214
215
216
	if (device->block) {
		rc = dasd_gendisk_alloc(device->block);
		if (rc)
			return rc;
	}
Linus Torvalds's avatar
Linus Torvalds committed
217
	/* register 'device' debug area, used for all DBF_DEV_XXX calls */
218
	device->debug_area = debug_register(dev_name(&device->cdev->dev), 1, 1,
219
					    8 * sizeof(long));
Linus Torvalds's avatar
Linus Torvalds committed
220
	debug_register_view(device->debug_area, &debug_sprintf_view);
221
	debug_set_level(device->debug_area, DBF_WARNING);
Linus Torvalds's avatar
Linus Torvalds committed
222
223
224
225
226
227
228
229
230
	DBF_DEV_EVENT(DBF_EMERG, device, "%s", "debug area created");

	device->state = DASD_STATE_BASIC;
	return 0;
}

/*
 * Release the irq line for the device. Terminate any running i/o.
 */
231
static int dasd_state_basic_to_known(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
232
{
233
	int rc;
234
235
236
237
238
	if (device->block) {
		dasd_gendisk_free(device->block);
		dasd_block_clear_timer(device->block);
	}
	rc = dasd_flush_device_queue(device);
239
240
	if (rc)
		return rc;
241
	dasd_device_clear_timer(device);
242

Linus Torvalds's avatar
Linus Torvalds committed
243
244
245
246
247
248
	DBF_DEV_EVENT(DBF_EMERG, device, "%p debug area deleted", device);
	if (device->debug_area != NULL) {
		debug_unregister(device->debug_area);
		device->debug_area = NULL;
	}
	device->state = DASD_STATE_KNOWN;
249
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
250
251
252
253
254
255
256
257
258
259
260
}

/*
 * Do the initial analysis. The do_analysis function may return
 * -EAGAIN in which case the device keeps the state DASD_STATE_BASIC
 * until the discipline decides to continue the startup sequence
 * by calling the function dasd_change_state. The eckd disciplines
 * uses this to start a ccw that detects the format. The completion
 * interrupt for this detection ccw uses the kernel event daemon to
 * trigger the call to dasd_change_state. All this is done in the
 * discipline code, see dasd_eckd.c.
261
262
263
264
 * After the analysis ccw is done (do_analysis returned 0) the block
 * device is setup.
 * In case the analysis returns an error, the device setup is stopped
 * (a fake disk was already added to allow formatting).
Linus Torvalds's avatar
Linus Torvalds committed
265
 */
266
static int dasd_state_basic_to_ready(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
267
268
{
	int rc;
269
	struct dasd_block *block;
Linus Torvalds's avatar
Linus Torvalds committed
270
271

	rc = 0;
272
	block = device->block;
273
	/* make disk known with correct capacity */
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
	if (block) {
		if (block->base->discipline->do_analysis != NULL)
			rc = block->base->discipline->do_analysis(block);
		if (rc) {
			if (rc != -EAGAIN)
				device->state = DASD_STATE_UNFMT;
			return rc;
		}
		dasd_setup_queue(block);
		set_capacity(block->gdp,
			     block->blocks << block->s2b_shift);
		device->state = DASD_STATE_READY;
		rc = dasd_scan_partitions(block);
		if (rc)
			device->state = DASD_STATE_BASIC;
	} else {
		device->state = DASD_STATE_READY;
	}
292
	return rc;
Linus Torvalds's avatar
Linus Torvalds committed
293
294
295
296
297
298
299
}

/*
 * Remove device from block device layer. Destroy dirty buffers.
 * Forget format information. Check if the target level is basic
 * and if it is create fake disk for formatting.
 */
300
static int dasd_state_ready_to_basic(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
301
{
302
303
	int rc;

Linus Torvalds's avatar
Linus Torvalds committed
304
	device->state = DASD_STATE_BASIC;
305
306
307
308
309
310
311
312
313
314
315
316
317
	if (device->block) {
		struct dasd_block *block = device->block;
		rc = dasd_flush_block_queue(block);
		if (rc) {
			device->state = DASD_STATE_READY;
			return rc;
		}
		dasd_destroy_partitions(block);
		dasd_flush_request_queue(block);
		block->blocks = 0;
		block->bp_block = 0;
		block->s2b_shift = 0;
	}
318
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
319
320
}

321
322
323
/*
 * Back to basic.
 */
324
static int dasd_state_unfmt_to_basic(struct dasd_device *device)
325
326
{
	device->state = DASD_STATE_BASIC;
327
	return 0;
328
329
}

Linus Torvalds's avatar
Linus Torvalds committed
330
331
332
333
334
/*
 * Make the device online and schedule the bottom half to start
 * the requeueing of requests from the linux request queue to the
 * ccw queue.
 */
335
static int
Linus Torvalds's avatar
Linus Torvalds committed
336
337
dasd_state_ready_to_online(struct dasd_device * device)
{
338
339
340
341
342
343
344
	int rc;

	if (device->discipline->ready_to_online) {
		rc = device->discipline->ready_to_online(device);
		if (rc)
			return rc;
	}
Linus Torvalds's avatar
Linus Torvalds committed
345
	device->state = DASD_STATE_ONLINE;
346
347
	if (device->block)
		dasd_schedule_block_bh(device->block);
Linus Torvalds's avatar
Linus Torvalds committed
348
349
350
351
352
353
	return 0;
}

/*
 * Stop the requeueing of requests again.
 */
354
static int dasd_state_online_to_ready(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
355
{
356
357
358
359
360
361
362
	int rc;

	if (device->discipline->online_to_ready) {
		rc = device->discipline->online_to_ready(device);
		if (rc)
			return rc;
	}
Linus Torvalds's avatar
Linus Torvalds committed
363
	device->state = DASD_STATE_READY;
364
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
365
366
367
368
369
}

/*
 * Device startup state changes.
 */
370
static int dasd_increase_state(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
{
	int rc;

	rc = 0;
	if (device->state == DASD_STATE_NEW &&
	    device->target >= DASD_STATE_KNOWN)
		rc = dasd_state_new_to_known(device);

	if (!rc &&
	    device->state == DASD_STATE_KNOWN &&
	    device->target >= DASD_STATE_BASIC)
		rc = dasd_state_known_to_basic(device);

	if (!rc &&
	    device->state == DASD_STATE_BASIC &&
	    device->target >= DASD_STATE_READY)
		rc = dasd_state_basic_to_ready(device);

389
390
391
392
393
	if (!rc &&
	    device->state == DASD_STATE_UNFMT &&
	    device->target > DASD_STATE_UNFMT)
		rc = -EPERM;

Linus Torvalds's avatar
Linus Torvalds committed
394
395
396
397
398
399
400
401
402
403
404
	if (!rc &&
	    device->state == DASD_STATE_READY &&
	    device->target >= DASD_STATE_ONLINE)
		rc = dasd_state_ready_to_online(device);

	return rc;
}

/*
 * Device shutdown state changes.
 */
405
static int dasd_decrease_state(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
406
{
407
408
409
	int rc;

	rc = 0;
Linus Torvalds's avatar
Linus Torvalds committed
410
411
	if (device->state == DASD_STATE_ONLINE &&
	    device->target <= DASD_STATE_READY)
412
		rc = dasd_state_online_to_ready(device);
413

414
415
	if (!rc &&
	    device->state == DASD_STATE_READY &&
Linus Torvalds's avatar
Linus Torvalds committed
416
	    device->target <= DASD_STATE_BASIC)
417
		rc = dasd_state_ready_to_basic(device);
418

419
420
	if (!rc &&
	    device->state == DASD_STATE_UNFMT &&
421
	    device->target <= DASD_STATE_BASIC)
422
		rc = dasd_state_unfmt_to_basic(device);
423

424
425
	if (!rc &&
	    device->state == DASD_STATE_BASIC &&
Linus Torvalds's avatar
Linus Torvalds committed
426
	    device->target <= DASD_STATE_KNOWN)
427
		rc = dasd_state_basic_to_known(device);
428

429
430
	if (!rc &&
	    device->state == DASD_STATE_KNOWN &&
Linus Torvalds's avatar
Linus Torvalds committed
431
	    device->target <= DASD_STATE_NEW)
432
		rc = dasd_state_known_to_new(device);
Linus Torvalds's avatar
Linus Torvalds committed
433

434
	return rc;
Linus Torvalds's avatar
Linus Torvalds committed
435
436
437
438
439
}

/*
 * This is the main startup/shutdown routine.
 */
440
static void dasd_change_state(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
{
        int rc;

	if (device->state == device->target)
		/* Already where we want to go today... */
		return;
	if (device->state < device->target)
		rc = dasd_increase_state(device);
	else
		rc = dasd_decrease_state(device);
        if (rc && rc != -EAGAIN)
                device->target = device->state;

	if (device->state == device->target)
		wake_up(&dasd_init_waitq);
456
457
458

	/* let user-space know that the device status changed */
	kobject_uevent(&device->cdev->dev.kobj, KOBJ_CHANGE);
Linus Torvalds's avatar
Linus Torvalds committed
459
460
461
462
463
464
465
466
}

/*
 * Kick starter for devices that did not complete the startup/shutdown
 * procedure or were sleeping because of a pending state.
 * dasd_kick_device will schedule a call do do_kick_device to the kernel
 * event daemon.
 */
467
static void do_kick_device(struct work_struct *work)
Linus Torvalds's avatar
Linus Torvalds committed
468
{
469
	struct dasd_device *device = container_of(work, struct dasd_device, kick_work);
Linus Torvalds's avatar
Linus Torvalds committed
470
	dasd_change_state(device);
471
	dasd_schedule_device_bh(device);
Linus Torvalds's avatar
Linus Torvalds committed
472
473
474
	dasd_put_device(device);
}

475
void dasd_kick_device(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
476
477
478
479
480
481
482
483
484
{
	dasd_get_device(device);
	/* queue call to dasd_kick_device to the kernel event daemon. */
	schedule_work(&device->kick_work);
}

/*
 * Set the target state for a device and starts the state change.
 */
485
void dasd_set_target_state(struct dasd_device *device, int target)
Linus Torvalds's avatar
Linus Torvalds committed
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
{
	/* If we are in probeonly mode stop at DASD_STATE_READY. */
	if (dasd_probeonly && target > DASD_STATE_READY)
		target = DASD_STATE_READY;
	if (device->target != target) {
                if (device->state == target)
			wake_up(&dasd_init_waitq);
		device->target = target;
	}
	if (device->state != device->target)
		dasd_change_state(device);
}

/*
 * Enable devices with device numbers in [from..to].
 */
502
static inline int _wait_for_device(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
503
504
505
506
{
	return (device->state == device->target);
}

507
void dasd_enable_device(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
{
	dasd_set_target_state(device, DASD_STATE_ONLINE);
	if (device->state <= DASD_STATE_KNOWN)
		/* No discipline for device found. */
		dasd_set_target_state(device, DASD_STATE_NEW);
	/* Now wait for the devices to come up. */
	wait_event(dasd_init_waitq, _wait_for_device(device));
}

/*
 * SECTION: device operation (interrupt handler, start i/o, term i/o ...)
 */
#ifdef CONFIG_DASD_PROFILE

struct dasd_profile_info_t dasd_global_profile;
unsigned int dasd_profile_level = DASD_PROFILE_OFF;

/*
 * Increments counter in global and local profiling structures.
 */
528
#define dasd_profile_counter(value, counter, block) \
Linus Torvalds's avatar
Linus Torvalds committed
529
530
531
532
{ \
	int index; \
	for (index = 0; index < 31 && value >> (2+index); index++); \
	dasd_global_profile.counter[index]++; \
533
	block->profile.counter[index]++; \
Linus Torvalds's avatar
Linus Torvalds committed
534
535
536
537
538
}

/*
 * Add profiling information for cqr before execution.
 */
539
540
541
static void dasd_profile_start(struct dasd_block *block,
			       struct dasd_ccw_req *cqr,
			       struct request *req)
Linus Torvalds's avatar
Linus Torvalds committed
542
543
544
545
546
547
548
549
550
{
	struct list_head *l;
	unsigned int counter;

	if (dasd_profile_level != DASD_PROFILE_ON)
		return;

	/* count the length of the chanq for statistics */
	counter = 0;
551
	list_for_each(l, &block->ccw_queue)
Linus Torvalds's avatar
Linus Torvalds committed
552
553
554
		if (++counter >= 31)
			break;
	dasd_global_profile.dasd_io_nr_req[counter]++;
555
	block->profile.dasd_io_nr_req[counter]++;
Linus Torvalds's avatar
Linus Torvalds committed
556
557
558
559
560
}

/*
 * Add profiling information for cqr after execution.
 */
561
562
563
static void dasd_profile_end(struct dasd_block *block,
			     struct dasd_ccw_req *cqr,
			     struct request *req)
Linus Torvalds's avatar
Linus Torvalds committed
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
{
	long strtime, irqtime, endtime, tottime;	/* in microseconds */
	long tottimeps, sectors;

	if (dasd_profile_level != DASD_PROFILE_ON)
		return;

	sectors = req->nr_sectors;
	if (!cqr->buildclk || !cqr->startclk ||
	    !cqr->stopclk || !cqr->endclk ||
	    !sectors)
		return;

	strtime = ((cqr->startclk - cqr->buildclk) >> 12);
	irqtime = ((cqr->stopclk - cqr->startclk) >> 12);
	endtime = ((cqr->endclk - cqr->stopclk) >> 12);
	tottime = ((cqr->endclk - cqr->buildclk) >> 12);
	tottimeps = tottime / sectors;

	if (!dasd_global_profile.dasd_io_reqs)
		memset(&dasd_global_profile, 0,
585
		       sizeof(struct dasd_profile_info_t));
Linus Torvalds's avatar
Linus Torvalds committed
586
587
588
	dasd_global_profile.dasd_io_reqs++;
	dasd_global_profile.dasd_io_sects += sectors;

589
590
591
592
593
	if (!block->profile.dasd_io_reqs)
		memset(&block->profile, 0,
		       sizeof(struct dasd_profile_info_t));
	block->profile.dasd_io_reqs++;
	block->profile.dasd_io_sects += sectors;
Linus Torvalds's avatar
Linus Torvalds committed
594

595
596
597
598
599
600
601
	dasd_profile_counter(sectors, dasd_io_secs, block);
	dasd_profile_counter(tottime, dasd_io_times, block);
	dasd_profile_counter(tottimeps, dasd_io_timps, block);
	dasd_profile_counter(strtime, dasd_io_time1, block);
	dasd_profile_counter(irqtime, dasd_io_time2, block);
	dasd_profile_counter(irqtime / sectors, dasd_io_time2ps, block);
	dasd_profile_counter(endtime, dasd_io_time3, block);
Linus Torvalds's avatar
Linus Torvalds committed
602
603
}
#else
604
605
#define dasd_profile_start(block, cqr, req) do {} while (0)
#define dasd_profile_end(block, cqr, req) do {} while (0)
Linus Torvalds's avatar
Linus Torvalds committed
606
607
608
609
610
611
612
613
614
#endif				/* CONFIG_DASD_PROFILE */

/*
 * Allocate memory for a channel program with 'cplength' channel
 * command words and 'datasize' additional space. There are two
 * variantes: 1) dasd_kmalloc_request uses kmalloc to get the needed
 * memory and 2) dasd_smalloc_request uses the static ccw memory
 * that gets allocated for each device.
 */
615
616
617
struct dasd_ccw_req *dasd_kmalloc_request(char *magic, int cplength,
					  int datasize,
					  struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
618
619
620
621
{
	struct dasd_ccw_req *cqr;

	/* Sanity checks */
622
623
	BUG_ON( magic == NULL || datasize > PAGE_SIZE ||
	     (cplength*sizeof(struct ccw1)) > PAGE_SIZE);
Linus Torvalds's avatar
Linus Torvalds committed
624

625
	cqr = kzalloc(sizeof(struct dasd_ccw_req), GFP_ATOMIC);
Linus Torvalds's avatar
Linus Torvalds committed
626
627
628
629
	if (cqr == NULL)
		return ERR_PTR(-ENOMEM);
	cqr->cpaddr = NULL;
	if (cplength > 0) {
630
		cqr->cpaddr = kcalloc(cplength, sizeof(struct ccw1),
Linus Torvalds's avatar
Linus Torvalds committed
631
632
633
634
635
636
637
638
				      GFP_ATOMIC | GFP_DMA);
		if (cqr->cpaddr == NULL) {
			kfree(cqr);
			return ERR_PTR(-ENOMEM);
		}
	}
	cqr->data = NULL;
	if (datasize > 0) {
639
		cqr->data = kzalloc(datasize, GFP_ATOMIC | GFP_DMA);
Linus Torvalds's avatar
Linus Torvalds committed
640
		if (cqr->data == NULL) {
641
			kfree(cqr->cpaddr);
Linus Torvalds's avatar
Linus Torvalds committed
642
643
644
645
646
647
648
649
650
651
652
			kfree(cqr);
			return ERR_PTR(-ENOMEM);
		}
	}
	strncpy((char *) &cqr->magic, magic, 4);
	ASCEBC((char *) &cqr->magic, 4);
	set_bit(DASD_CQR_FLAGS_USE_ERP, &cqr->flags);
	dasd_get_device(device);
	return cqr;
}

653
654
655
struct dasd_ccw_req *dasd_smalloc_request(char *magic, int cplength,
					  int datasize,
					  struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
656
657
658
659
660
661
662
{
	unsigned long flags;
	struct dasd_ccw_req *cqr;
	char *data;
	int size;

	/* Sanity checks */
663
664
	BUG_ON( magic == NULL || datasize > PAGE_SIZE ||
	     (cplength*sizeof(struct ccw1)) > PAGE_SIZE);
Linus Torvalds's avatar
Linus Torvalds committed
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701

	size = (sizeof(struct dasd_ccw_req) + 7L) & -8L;
	if (cplength > 0)
		size += cplength * sizeof(struct ccw1);
	if (datasize > 0)
		size += datasize;
	spin_lock_irqsave(&device->mem_lock, flags);
	cqr = (struct dasd_ccw_req *)
		dasd_alloc_chunk(&device->ccw_chunks, size);
	spin_unlock_irqrestore(&device->mem_lock, flags);
	if (cqr == NULL)
		return ERR_PTR(-ENOMEM);
	memset(cqr, 0, sizeof(struct dasd_ccw_req));
	data = (char *) cqr + ((sizeof(struct dasd_ccw_req) + 7L) & -8L);
	cqr->cpaddr = NULL;
	if (cplength > 0) {
		cqr->cpaddr = (struct ccw1 *) data;
		data += cplength*sizeof(struct ccw1);
		memset(cqr->cpaddr, 0, cplength*sizeof(struct ccw1));
	}
	cqr->data = NULL;
	if (datasize > 0) {
		cqr->data = data;
 		memset(cqr->data, 0, datasize);
	}
	strncpy((char *) &cqr->magic, magic, 4);
	ASCEBC((char *) &cqr->magic, 4);
	set_bit(DASD_CQR_FLAGS_USE_ERP, &cqr->flags);
	dasd_get_device(device);
	return cqr;
}

/*
 * Free memory of a channel program. This function needs to free all the
 * idal lists that might have been created by dasd_set_cda and the
 * struct dasd_ccw_req itself.
 */
702
void dasd_kfree_request(struct dasd_ccw_req *cqr, struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
703
{
704
#ifdef CONFIG_64BIT
Linus Torvalds's avatar
Linus Torvalds committed
705
706
707
708
709
710
711
712
	struct ccw1 *ccw;

	/* Clear any idals used for the request. */
	ccw = cqr->cpaddr;
	do {
		clear_normalized_cda(ccw);
	} while (ccw++->flags & (CCW_FLAG_CC | CCW_FLAG_DC));
#endif
713
714
	kfree(cqr->cpaddr);
	kfree(cqr->data);
Linus Torvalds's avatar
Linus Torvalds committed
715
716
717
718
	kfree(cqr);
	dasd_put_device(device);
}

719
void dasd_sfree_request(struct dasd_ccw_req *cqr, struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
720
721
722
723
724
725
726
727
728
729
730
731
{
	unsigned long flags;

	spin_lock_irqsave(&device->mem_lock, flags);
	dasd_free_chunk(&device->ccw_chunks, cqr);
	spin_unlock_irqrestore(&device->mem_lock, flags);
	dasd_put_device(device);
}

/*
 * Check discipline magic in cqr.
 */
732
static inline int dasd_check_cqr(struct dasd_ccw_req *cqr)
Linus Torvalds's avatar
Linus Torvalds committed
733
734
735
736
737
{
	struct dasd_device *device;

	if (cqr == NULL)
		return -EINVAL;
738
	device = cqr->startdev;
Linus Torvalds's avatar
Linus Torvalds committed
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
	if (strncmp((char *) &cqr->magic, device->discipline->ebcname, 4)) {
		DEV_MESSAGE(KERN_WARNING, device,
			    " dasd_ccw_req 0x%08x magic doesn't match"
			    " discipline 0x%08x",
			    cqr->magic,
			    *(unsigned int *) device->discipline->name);
		return -EINVAL;
	}
	return 0;
}

/*
 * Terminate the current i/o and set the request to clear_pending.
 * Timer keeps device runnig.
 * ccw_device_clear can fail if the i/o subsystem
 * is in a bad mood.
 */
756
int dasd_term_IO(struct dasd_ccw_req *cqr)
Linus Torvalds's avatar
Linus Torvalds committed
757
758
759
760
761
762
763
764
765
{
	struct dasd_device *device;
	int retries, rc;

	/* Check the cqr */
	rc = dasd_check_cqr(cqr);
	if (rc)
		return rc;
	retries = 0;
766
	device = (struct dasd_device *) cqr->startdev;
Linus Torvalds's avatar
Linus Torvalds committed
767
768
769
770
	while ((retries < 5) && (cqr->status == DASD_CQR_IN_IO)) {
		rc = ccw_device_clear(device->cdev, (long) cqr);
		switch (rc) {
		case 0:	/* termination successful */
771
			cqr->retries--;
772
			cqr->status = DASD_CQR_CLEAR_PENDING;
Linus Torvalds's avatar
Linus Torvalds committed
773
			cqr->stopclk = get_clock();
774
			cqr->starttime = 0;
Linus Torvalds's avatar
Linus Torvalds committed
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
			DBF_DEV_EVENT(DBF_DEBUG, device,
				      "terminate cqr %p successful",
				      cqr);
			break;
		case -ENODEV:
			DBF_DEV_EVENT(DBF_ERR, device, "%s",
				      "device gone, retry");
			break;
		case -EIO:
			DBF_DEV_EVENT(DBF_ERR, device, "%s",
				      "I/O error, retry");
			break;
		case -EINVAL:
		case -EBUSY:
			DBF_DEV_EVENT(DBF_ERR, device, "%s",
				      "device busy, retry later");
			break;
		default:
			DEV_MESSAGE(KERN_ERR, device,
				    "line %d unknown RC=%d, please "
				    "report to linux390@de.ibm.com",
				    __LINE__, rc);
			BUG();
			break;
		}
		retries++;
	}
802
	dasd_schedule_device_bh(device);
Linus Torvalds's avatar
Linus Torvalds committed
803
804
805
806
807
808
809
	return rc;
}

/*
 * Start the i/o. This start_IO can fail if the channel is really busy.
 * In that case set up a timer to start the request later.
 */
810
int dasd_start_IO(struct dasd_ccw_req *cqr)
Linus Torvalds's avatar
Linus Torvalds committed
811
812
813
814
815
816
817
818
{
	struct dasd_device *device;
	int rc;

	/* Check the cqr */
	rc = dasd_check_cqr(cqr);
	if (rc)
		return rc;
819
	device = (struct dasd_device *) cqr->startdev;
Linus Torvalds's avatar
Linus Torvalds committed
820
821
822
823
	if (cqr->retries < 0) {
		DEV_MESSAGE(KERN_DEBUG, device,
			    "start_IO: request %p (%02x/%i) - no retry left.",
			    cqr, cqr->status, cqr->retries);
824
		cqr->status = DASD_CQR_ERROR;
Linus Torvalds's avatar
Linus Torvalds committed
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
		return -EIO;
	}
	cqr->startclk = get_clock();
	cqr->starttime = jiffies;
	cqr->retries--;
	rc = ccw_device_start(device->cdev, cqr->cpaddr, (long) cqr,
			      cqr->lpm, 0);
	switch (rc) {
	case 0:
		cqr->status = DASD_CQR_IN_IO;
		DBF_DEV_EVENT(DBF_DEBUG, device,
			      "start_IO: request %p started successful",
			      cqr);
		break;
	case -EBUSY:
		DBF_DEV_EVENT(DBF_ERR, device, "%s",
			      "start_IO: device busy, retry later");
		break;
	case -ETIMEDOUT:
		DBF_DEV_EVENT(DBF_ERR, device, "%s",
			      "start_IO: request timeout, retry later");
		break;
	case -EACCES:
		/* -EACCES indicates that the request used only a
		 * subset of the available pathes and all these
		 * pathes are gone.
		 * Do a retry with all available pathes.
		 */
		cqr->lpm = LPM_ANYPATH;
		DBF_DEV_EVENT(DBF_ERR, device, "%s",
			      "start_IO: selected pathes gone,"
			      " retry on all pathes");
		break;
	case -ENODEV:
	case -EIO:
		DBF_DEV_EVENT(DBF_ERR, device, "%s",
			      "start_IO: device gone, retry");
		break;
	default:
		DEV_MESSAGE(KERN_ERR, device,
			    "line %d unknown RC=%d, please report"
			    " to linux390@de.ibm.com", __LINE__, rc);
		BUG();
		break;
	}
	return rc;
}

/*
 * Timeout function for dasd devices. This is used for different purposes
 *  1) missing interrupt handler for normal operation
 *  2) delayed start of request where start_IO failed with -EBUSY
 *  3) timeout for missing state change interrupts
 * The head of the ccw queue will have status DASD_CQR_IN_IO for 1),
 * DASD_CQR_QUEUED for 2) and 3).
 */
881
static void dasd_device_timeout(unsigned long ptr)
Linus Torvalds's avatar
Linus Torvalds committed
882
883
884
885
886
887
888
889
890
{
	unsigned long flags;
	struct dasd_device *device;

	device = (struct dasd_device *) ptr;
	spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags);
	/* re-activate request queue */
        device->stopped &= ~DASD_STOPPED_PENDING;
	spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags);
891
	dasd_schedule_device_bh(device);
Linus Torvalds's avatar
Linus Torvalds committed
892
893
894
895
896
}

/*
 * Setup timeout for a device in jiffies.
 */
897
void dasd_device_set_timer(struct dasd_device *device, int expires)
Linus Torvalds's avatar
Linus Torvalds committed
898
899
900
901
902
903
904
905
906
907
{
	if (expires == 0) {
		if (timer_pending(&device->timer))
			del_timer(&device->timer);
		return;
	}
	if (timer_pending(&device->timer)) {
		if (mod_timer(&device->timer, jiffies + expires))
			return;
	}
908
	device->timer.function = dasd_device_timeout;
Linus Torvalds's avatar
Linus Torvalds committed
909
910
911
912
913
914
915
916
	device->timer.data = (unsigned long) device;
	device->timer.expires = jiffies + expires;
	add_timer(&device->timer);
}

/*
 * Clear timeout for a device.
 */
917
void dasd_device_clear_timer(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
918
919
920
921
922
{
	if (timer_pending(&device->timer))
		del_timer(&device->timer);
}

923
924
static void dasd_handle_killed_request(struct ccw_device *cdev,
				       unsigned long intparm)
Linus Torvalds's avatar
Linus Torvalds committed
925
926
927
928
{
	struct dasd_ccw_req *cqr;
	struct dasd_device *device;

929
930
	if (!intparm)
		return;
Linus Torvalds's avatar
Linus Torvalds committed
931
932
933
934
935
	cqr = (struct dasd_ccw_req *) intparm;
	if (cqr->status != DASD_CQR_IN_IO) {
		MESSAGE(KERN_DEBUG,
			"invalid status in handle_killed_request: "
			"bus_id %s, status %02x",
936
			dev_name(&cdev->dev), cqr->status);
Linus Torvalds's avatar
Linus Torvalds committed
937
938
939
		return;
	}

940
	device = (struct dasd_device *) cqr->startdev;
Linus Torvalds's avatar
Linus Torvalds committed
941
	if (device == NULL ||
942
	    device != dasd_device_from_cdev_locked(cdev) ||
Linus Torvalds's avatar
Linus Torvalds committed
943
944
	    strncmp(device->discipline->ebcname, (char *) &cqr->magic, 4)) {
		MESSAGE(KERN_DEBUG, "invalid device in request: bus_id %s",
945
			dev_name(&cdev->dev));
Linus Torvalds's avatar
Linus Torvalds committed
946
947
948
949
950
951
		return;
	}

	/* Schedule request to be retried. */
	cqr->status = DASD_CQR_QUEUED;

952
953
	dasd_device_clear_timer(device);
	dasd_schedule_device_bh(device);
Linus Torvalds's avatar
Linus Torvalds committed
954
955
956
	dasd_put_device(device);
}

957
void dasd_generic_handle_state_change(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
958
{
959
960
961
	/* First of all start sense subsystem status request. */
	dasd_eer_snss(device);

Linus Torvalds's avatar
Linus Torvalds committed
962
	device->stopped &= ~DASD_STOPPED_PENDING;
963
964
965
	dasd_schedule_device_bh(device);
	if (device->block)
		dasd_schedule_block_bh(device->block);
Linus Torvalds's avatar
Linus Torvalds committed
966
967
968
969
970
}

/*
 * Interrupt handler for "normal" ssch-io based dasd devices.
 */
971
972
void dasd_int_handler(struct ccw_device *cdev, unsigned long intparm,
		      struct irb *irb)
Linus Torvalds's avatar
Linus Torvalds committed
973
974
975
976
977
978
979
980
981
982
983
984
{
	struct dasd_ccw_req *cqr, *next;
	struct dasd_device *device;
	unsigned long long now;
	int expires;

	if (IS_ERR(irb)) {
		switch (PTR_ERR(irb)) {
		case -EIO:
			break;
		case -ETIMEDOUT:
			printk(KERN_WARNING"%s(%s): request timed out\n",
985
			       __func__, dev_name(&cdev->dev));
Linus Torvalds's avatar
Linus Torvalds committed
986
987
988
			break;
		default:
			printk(KERN_WARNING"%s(%s): unknown error %ld\n",
989
			       __func__, dev_name(&cdev->dev), PTR_ERR(irb));
Linus Torvalds's avatar
Linus Torvalds committed
990
		}
991
		dasd_handle_killed_request(cdev, intparm);
Linus Torvalds's avatar
Linus Torvalds committed
992
993
994
995
996
997
		return;
	}

	now = get_clock();

	DBF_EVENT(DBF_ERR, "Interrupt: bus_id %s CS/DS %04x ip %08x",
998
		  dev_name(&cdev->dev), ((irb->scsw.cmd.cstat << 8) |
999
		  irb->scsw.cmd.dstat), (unsigned int) intparm);
Linus Torvalds's avatar
Linus Torvalds committed
1000

1001
1002
	/* check for unsolicited interrupts */
	cqr = (struct dasd_ccw_req *) intparm;
1003
1004
1005
	if (!cqr || ((irb->scsw.cmd.cc == 1) &&
		     (irb->scsw.cmd.fctl & SCSW_FCTL_START_FUNC) &&
		     (irb->scsw.cmd.stctl & SCSW_STCTL_STATUS_PEND))) {
1006
1007
		if (cqr && cqr->status == DASD_CQR_IN_IO)
			cqr->status = DASD_CQR_QUEUED;
1008
		device = dasd_device_from_cdev_locked(cdev);
Linus Torvalds's avatar
Linus Torvalds committed
1009
		if (!IS_ERR(device)) {
1010
1011
1012
			dasd_device_clear_timer(device);
			device->discipline->handle_unsolicited_interrupt(device,
									 irb);
Linus Torvalds's avatar
Linus Torvalds committed
1013
1014
1015
1016
1017
			dasd_put_device(device);
		}
		return;
	}

1018
1019
	device = (struct dasd_device *) cqr->startdev;
	if (!device ||
Linus Torvalds's avatar
Linus Torvalds committed
1020
1021
	    strncmp(device->discipline->ebcname, (char *) &cqr->magic, 4)) {
		MESSAGE(KERN_DEBUG, "invalid device in request: bus_id %s",
1022
			dev_name(&cdev->dev));
Linus Torvalds's avatar
Linus Torvalds committed
1023
1024
1025
1026
		return;
	}

	/* Check for clear pending */
1027
	if (cqr->status == DASD_CQR_CLEAR_PENDING &&
1028
	    irb->scsw.cmd.fctl & SCSW_FCTL_CLEAR_FUNC) {
1029
1030
		cqr->status = DASD_CQR_CLEARED;
		dasd_device_clear_timer(device);
1031
		wake_up(&dasd_flush_wq);
1032
		dasd_schedule_device_bh(device);
Linus Torvalds's avatar
Linus Torvalds committed
1033
1034
1035
1036
1037
1038
1039
		return;
	}

 	/* check status - the request might have been killed by dyn detach */
	if (cqr->status != DASD_CQR_IN_IO) {
		MESSAGE(KERN_DEBUG,
			"invalid status: bus_id %s, status %02x",
1040
			dev_name(&cdev->dev), cqr->status);
Linus Torvalds's avatar
Linus Torvalds committed
1041
1042
1043
		return;
	}
	DBF_DEV_EVENT(DBF_DEBUG, device, "Int: CS/DS 0x%04x for cqr %p",
1044
		      ((irb->scsw.cmd.cstat << 8) | irb->scsw.cmd.dstat), cqr);
1045
	next = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
1046
	expires = 0;
1047
1048
	if (irb->scsw.cmd.dstat == (DEV_STAT_CHN_END | DEV_STAT_DEV_END) &&
	    irb->scsw.cmd.cstat == 0 && !irb->esw.esw0.erw.cons) {
1049
1050
		/* request was completed successfully */
		cqr->status = DASD_CQR_SUCCESS;
Linus Torvalds's avatar
Linus Torvalds committed
1051
1052
		cqr->stopclk = now;
		/* Start first request on queue if possible -> fast_io. */
1053
1054
1055
		if (cqr->devlist.next != &device->ccw_queue) {
			next = list_entry(cqr->devlist.next,
					  struct dasd_ccw_req, devlist);
Linus Torvalds's avatar
Linus Torvalds committed
1056
		}
1057
1058
	} else {  /* error */
		memcpy(&cqr->irb, irb, sizeof(struct irb));
1059
1060
1061
		if (device->features & DASD_FEATURE_ERPLOG) {
			dasd_log_sense(cqr, irb);
		}
1062
1063
1064
		/*
		 * If we don't want complex ERP for this request, then just
		 * reset this and retry it in the fastpath
1065
		 */
1066
		if (!test_bit(DASD_CQR_FLAGS_USE_ERP, &cqr->flags) &&
1067
1068
1069
1070
1071
1072
1073
1074
		    cqr->retries > 0) {
			DEV_MESSAGE(KERN_DEBUG, device,
				    "default ERP in fastpath (%i retries left)",
				    cqr->retries);
			cqr->lpm    = LPM_ANYPATH;
			cqr->status = DASD_CQR_QUEUED;
			next = cqr;
		} else
Linus Torvalds's avatar
Linus Torvalds committed
1075
			cqr->status = DASD_CQR_ERROR;
1076
1077
1078
1079
1080
1081
1082
1083
1084
	}
	if (next && (next->status == DASD_CQR_QUEUED) &&
	    (!device->stopped)) {
		if (device->discipline->start_IO(next) == 0)
			expires = next->expires;
		else
			DEV_MESSAGE(KERN_DEBUG, device, "%s",
				    "Interrupt fastpath "
				    "failed!");
Linus Torvalds's avatar
Linus Torvalds committed
1085
1086
	}
	if (expires != 0)
1087
		dasd_device_set_timer(device, expires);
Linus Torvalds's avatar
Linus Torvalds committed
1088
	else
1089
1090
		dasd_device_clear_timer(device);
	dasd_schedule_device_bh(device);
Linus Torvalds's avatar
Linus Torvalds committed
1091
1092
1093
}

/*
1094
1095
 * If we have an error on a dasd_block layer request then we cancel
 * and return all further requests from the same dasd_block as well.
Linus Torvalds's avatar
Linus Torvalds committed
1096
 */
1097
1098
static void __dasd_device_recovery(struct dasd_device *device,
				   struct dasd_ccw_req *ref_cqr)
Linus Torvalds's avatar
Linus Torvalds committed
1099
{
1100
1101
	struct list_head *l, *n;
	struct dasd_ccw_req *cqr;
Linus Torvalds's avatar
Linus Torvalds committed
1102

1103
1104
1105
1106
1107
	/*
	 * only requeue request that came from the dasd_block layer
	 */
	if (!ref_cqr->block)
		return;
Linus Torvalds's avatar
Linus Torvalds committed
1108

1109
1110
1111
1112
1113
1114
1115
1116
	list_for_each_safe(l, n, &device->ccw_queue) {
		cqr = list_entry(l, struct dasd_ccw_req, devlist);
		if (cqr->status == DASD_CQR_QUEUED &&
		    ref_cqr->block == cqr->block) {
			cqr->status = DASD_CQR_CLEARED;
		}
	}
};
Linus Torvalds's avatar
Linus Torvalds committed
1117
1118

/*
1119
1120
 * Remove those ccw requests from the queue that need to be returned
 * to the upper layer.
Linus Torvalds's avatar
Linus Torvalds committed
1121
 */
1122
1123
static void __dasd_device_process_ccw_queue(struct dasd_device *device,
					    struct list_head *final_queue)
Linus Torvalds's avatar
Linus Torvalds committed
1124
1125
1126
1127
1128
1129
{
	struct list_head *l, *n;
	struct dasd_ccw_req *cqr;

	/* Process request with final status. */
	list_for_each_safe(l, n, &device->ccw_queue) {
1130
1131
		cqr = list_entry(l, struct dasd_ccw_req, devlist);

Linus Torvalds's avatar
Linus Torvalds committed
1132
		/* Stop list processing at the first non-final request. */
1133
1134
1135
		if (cqr->status == DASD_CQR_QUEUED ||
		    cqr->status == DASD_CQR_IN_IO ||
		    cqr->status == DASD_CQR_CLEAR_PENDING)
Linus Torvalds's avatar
Linus Torvalds committed
1136
1137
			break;
		if (cqr->status == DASD_CQR_ERROR) {
1138
			__dasd_device_recovery(device, cqr);
1139
		}
Linus Torvalds's avatar
Linus Torvalds committed
1140
		/* Rechain finished requests to final queue */
1141
		list_move_tail(&cqr->devlist, final_queue);
Linus Torvalds's avatar
Linus Torvalds committed
1142
1143
1144
1145
	}
}

/*
1146
1147
 * the cqrs from the final queue are returned to the upper layer
 * by setting a dasd_block state and calling the callback function
Linus Torvalds's avatar
Linus Torvalds committed
1148
 */
1149
1150
static void __dasd_device_process_final_queue(struct dasd_device *device,
					      struct list_head *final_queue)
Linus Torvalds's avatar
Linus Torvalds committed
1151
{
1152
	struct list_head *l, *n;
Linus Torvalds's avatar
Linus Torvalds committed
1153
	struct dasd_ccw_req *cqr;
1154
	struct dasd_block *block;
1155
1156
	void (*callback)(struct dasd_ccw_req *, void *data);
	void *callback_data;
1157

1158
1159
1160
	list_for_each_safe(l, n, final_queue) {
		cqr = list_entry(l, struct dasd_ccw_req, devlist);
		list_del_init(&cqr->devlist);
1161
		block = cqr->block;
1162
1163
		callback = cqr->callback;
		callback_data = cqr->callback_data;
1164
1165
		if (block)
			spin_lock_bh(&block->queue_lock);
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
		switch (cqr->status) {
		case DASD_CQR_SUCCESS:
			cqr->status = DASD_CQR_DONE;
			break;
		case DASD_CQR_ERROR:
			cqr->status = DASD_CQR_NEED_ERP;
			break;
		case DASD_CQR_CLEARED:
			cqr->status = DASD_CQR_TERMINATED;
			break;
		default:
			DEV_MESSAGE(KERN_ERR, device,
				    "wrong cqr status in __dasd_process_final_queue "
				    "for cqr %p, status %x",
				    cqr, cqr->status);
			BUG();
Linus Torvalds's avatar
Linus Torvalds committed
1182
		}
1183
		if (cqr->callback != NULL)
1184
			(callback)(cqr, callback_data);
1185
1186
		if (block)
			spin_unlock_bh(&block->queue_lock);
Linus Torvalds's avatar
Linus Torvalds committed
1187
1188
1189
1190
1191
1192
1193
	}
}

/*
 * Take a look at the first request on the ccw queue and check
 * if it reached its expire time. If so, terminate the IO.
 */
1194
static void __dasd_device_check_expire(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
1195
1196
1197
1198
1199
{
	struct dasd_ccw_req *cqr;

	if (list_empty(&device->ccw_queue))
		return;
1200
	cqr = list_entry(device->ccw_queue.next, struct dasd_ccw_req, devlist);
1201
1202
1203
1204
1205
1206
1207
1208
1209
	if ((cqr->status == DASD_CQR_IN_IO && cqr->expires != 0) &&
	    (time_after_eq(jiffies, cqr->expires + cqr->starttime))) {
		if (device->discipline->term_IO(cqr) != 0) {
			/* Hmpf, try again in 5 sec */
			DEV_MESSAGE(KERN_ERR, device,
				    "internal error - timeout (%is) expired "
				    "for cqr %p, termination failed, "
				    "retrying in 5s",
				    (cqr->expires/HZ), cqr);
1210
1211
			cqr->expires += 5*HZ;
			dasd_device_set_timer(device, 5*HZ);
1212
		} else {
1213
1214
1215
1216
			DEV_MESSAGE(KERN_ERR, device,
				    "internal error - timeout (%is) expired "
				    "for cqr %p (%i retries left)",
				    (cqr->expires/HZ), cqr, cqr->retries);
Linus Torvalds's avatar
Linus Torvalds committed
1217
1218
1219
1220
1221
1222
1223
1224
		}
	}
}

/*
 * Take a look at the first request on the ccw queue and check
 * if it needs to be started.
 */
1225
static void __dasd_device_start_head(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
1226
1227
1228
1229
1230
1231
{
	struct dasd_ccw_req *cqr;
	int rc;

	if (list_empty(&device->ccw_queue))
		return;
1232
	cqr = list_entry(device->ccw_queue.next, struct dasd_ccw_req, devlist);
1233
1234
	if (cqr->status != DASD_CQR_QUEUED)
		return;
1235
1236
1237
1238
	/* when device is stopped, return request to previous layer */
	if (device->stopped) {
		cqr->status = DASD_CQR_CLEARED;
		dasd_schedule_device_bh(device);
1239
		return;
1240
	}
1241
1242
1243

	rc = device->discipline->start_IO(cqr);
	if (rc == 0)
1244
		dasd_device_set_timer(device, cqr->expires);
1245
	else if (rc == -EACCES) {
1246
		dasd_schedule_device_bh(device);
1247
1248
	} else
		/* Hmpf, try again in 1/2 sec */
1249
		dasd_device_set_timer(device, 50);
1250
1251
}

Linus Torvalds's avatar
Linus Torvalds committed
1252
/*
1253
1254
1255
1256
1257
1258
1259
1260
 * Go through all request on the dasd_device request queue,
 * terminate them on the cdev if necessary, and return them to the
 * submitting layer via callback.
 * Note:
 * Make sure that all 'submitting layers' still exist when
 * this function is called!. In other words, when 'device' is a base
 * device then all block layer requests must have been removed before
 * via dasd_flush_block_queue.
Linus Torvalds's avatar
Linus Torvalds committed
1261
 */
1262
int dasd_flush_device_queue(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
1263
{
1264
1265
	struct dasd_ccw_req *cqr, *n;
	int rc;
Linus Torvalds's avatar
Linus Torvalds committed
1266
1267
1268
1269
	struct list_head flush_queue;

	INIT_LIST_HEAD(&flush_queue);
	spin_lock_irq(get_ccwdev_lock(device->cdev));
1270
	rc = 0;
1271
	list_for_each_entry_safe(cqr, n, &device->ccw_queue, devlist) {
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
		/* Check status and move request to flush_queue */
		switch (cqr->status) {
		case DASD_CQR_IN_IO:
			rc = device->discipline->term_IO(cqr);
			if (rc) {
				/* unable to terminate requeust */
				DEV_MESSAGE(KERN_ERR, device,
					    "dasd flush ccw_queue is unable "
					    " to terminate request %p",
					    cqr);
				/* stop flush processing */
				goto finished;
			}
			break;
		case DASD_CQR_QUEUED:
Linus Torvalds's avatar
Linus Torvalds committed
1287
			cqr->stopclk = get_clock();
1288
			cqr->status = DASD_CQR_CLEARED;
1289
			break;
1290
		default: /* no need to modify the others */
1291
1292
			break;
		}
1293
		list_move_tail(&cqr->devlist, &flush_queue);
1294
1295
1296
	}
finished:
	spin_unlock_irq(get_ccwdev_lock(device->cdev));
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
	/*
	 * After this point all requests must be in state CLEAR_PENDING,
	 * CLEARED, SUCCESS or ERROR. Now wait for CLEAR_PENDING to become
	 * one of the others.
	 */
	list_for_each_entry_safe(cqr, n, &flush_queue, devlist)
		wait_event(dasd_flush_wq,
			   (cqr->status != DASD_CQR_CLEAR_PENDING));
	/*
	 * Now set each request back to TERMINATED, DONE or NEED_ERP
	 * and call the callback function of flushed requests
	 */
	__dasd_device_process_final_queue(device, &flush_queue);
1310
	return rc;
Linus Torvalds's avatar
Linus Torvalds committed
1311
1312
1313
1314
1315
}

/*
 * Acquire the device lock and process queues for the device.
 */
1316
static void dasd_device_tasklet(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
1317
1318
1319
1320
1321
1322
1323
{
	struct list_head final_queue;

	atomic_set (&device->tasklet_scheduled, 0);
	INIT_LIST_HEAD(&final_queue);
	spin_lock_irq(get_ccwdev_lock(device->cdev));
	/* Check expire time of first request on the ccw queue. */
1324
1325
1326
	__dasd_device_check_expire(device);
	/* find final requests on ccw queue */
	__dasd_device_process_ccw_queue(device, &final_queue);
Linus Torvalds's avatar
Linus Torvalds committed
1327
1328
	spin_unlock_irq(get_ccwdev_lock(device->cdev));
	/* Now call the callback function of requests with final status */
1329
1330
	__dasd_device_process_final_queue(device, &final_queue);
	spin_lock_irq(get_ccwdev_lock(device->cdev));
Linus Torvalds's avatar
Linus Torvalds committed
1331
	/* Now check if the head of the ccw queue needs to be started. */
1332
1333
	__dasd_device_start_head(device);
	spin_unlock_irq(get_ccwdev_lock(device->cdev));
Linus Torvalds's avatar
Linus Torvalds committed
1334
1335
1336
1337
1338
1339
	dasd_put_device(device);
}

/*
 * Schedules a call to dasd_tasklet over the device tasklet.
 */
1340
void dasd_schedule_device_bh(struct dasd_device *device)
Linus Torvalds's avatar
Linus Torvalds committed
1341
1342
{
	/* Protect against rescheduling. */
1343
	if (atomic_cmpxchg (&device->tasklet_scheduled, 0, 1) != 0)
Linus Torvalds's avatar
Linus Torvalds committed
1344
1345
1346
1347
1348
1349
		return;
	dasd_get_device(device);
	tasklet_hi_schedule(&device->tasklet);
}

/*
1350
1351
 * Queue a request to the head of the device ccw_queue.
 * Start the I/O if possible.
Linus Torvalds's avatar
Linus Torvalds committed
1352
 */
1353
void dasd_add_request_head(struct dasd_ccw_req *cqr)
Linus Torvalds's avatar
Linus Torvalds committed
1354
1355
1356
1357
{
	struct dasd_device *device;
	unsigned long flags;

1358
	device = cqr->startdev;
Linus Torvalds's avatar
Linus Torvalds committed
1359
	spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags);
1360
1361
	cqr->status = DASD_CQR_QUEUED;
	list_add(&cqr->devlist, &device->ccw_queue);
Linus Torvalds's avatar
Linus Torvalds committed
1362
	/* let the bh start the request to keep them in order */
1363
	dasd_schedule_device_bh(device);
Linus Torvalds's avatar
Linus Torvalds committed
1364
1365
1366
1367
	spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags);
}

/*
1368
1369
 * Queue a request to the tail of the device ccw_queue.
 * Start the I/O if possible.
Linus Torvalds's avatar
Linus Torvalds committed
1370
 */