sdhci.c 106 KB
Newer Older
1
/*
Pierre Ossman's avatar
Pierre Ossman committed
2
 *  linux/drivers/mmc/host/sdhci.c - Secure Digital Host Controller Interface driver
3
 *
4
 *  Copyright (C) 2005-2008 Pierre Ossman, All Rights Reserved.
5
6
 *
 * This program is free software; you can redistribute it and/or modify
7
8
9
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or (at
 * your option) any later version.
10
11
12
13
 *
 * Thanks to the following companies for their support:
 *
 *     - JMicron (hardware and technical support)
14
15
16
 */

#include <linux/delay.h>
17
#include <linux/ktime.h>
18
#include <linux/highmem.h>
19
#include <linux/io.h>
20
#include <linux/module.h>
21
#include <linux/dma-mapping.h>
22
#include <linux/slab.h>
23
#include <linux/scatterlist.h>
24
#include <linux/sizes.h>
25
#include <linux/swiotlb.h>
26
#include <linux/regulator/consumer.h>
27
#include <linux/pm_runtime.h>
28
#include <linux/of.h>
29

30
31
#include <linux/leds.h>

32
#include <linux/mmc/mmc.h>
33
#include <linux/mmc/host.h>
34
#include <linux/mmc/card.h>
35
#include <linux/mmc/sdio.h>
36
#include <linux/mmc/slot-gpio.h>
37
38
39
40
41
42

#include "sdhci.h"

#define DRIVER_NAME "sdhci"

#define DBG(f, x...) \
43
	pr_debug("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
44

45
46
47
#define SDHCI_DUMP(f, x...) \
	pr_err("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)

48
49
#define MAX_TUNING_LOOP 40

50
static unsigned int debug_quirks = 0;
51
static unsigned int debug_quirks2;
52

53
54
static void sdhci_finish_data(struct sdhci_host *);

55
static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable);
56

57
void sdhci_dumpregs(struct sdhci_host *host)
58
{
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
	SDHCI_DUMP("============ SDHCI REGISTER DUMP ===========\n");

	SDHCI_DUMP("Sys addr:  0x%08x | Version:  0x%08x\n",
		   sdhci_readl(host, SDHCI_DMA_ADDRESS),
		   sdhci_readw(host, SDHCI_HOST_VERSION));
	SDHCI_DUMP("Blk size:  0x%08x | Blk cnt:  0x%08x\n",
		   sdhci_readw(host, SDHCI_BLOCK_SIZE),
		   sdhci_readw(host, SDHCI_BLOCK_COUNT));
	SDHCI_DUMP("Argument:  0x%08x | Trn mode: 0x%08x\n",
		   sdhci_readl(host, SDHCI_ARGUMENT),
		   sdhci_readw(host, SDHCI_TRANSFER_MODE));
	SDHCI_DUMP("Present:   0x%08x | Host ctl: 0x%08x\n",
		   sdhci_readl(host, SDHCI_PRESENT_STATE),
		   sdhci_readb(host, SDHCI_HOST_CONTROL));
	SDHCI_DUMP("Power:     0x%08x | Blk gap:  0x%08x\n",
		   sdhci_readb(host, SDHCI_POWER_CONTROL),
		   sdhci_readb(host, SDHCI_BLOCK_GAP_CONTROL));
	SDHCI_DUMP("Wake-up:   0x%08x | Clock:    0x%08x\n",
		   sdhci_readb(host, SDHCI_WAKE_UP_CONTROL),
		   sdhci_readw(host, SDHCI_CLOCK_CONTROL));
	SDHCI_DUMP("Timeout:   0x%08x | Int stat: 0x%08x\n",
		   sdhci_readb(host, SDHCI_TIMEOUT_CONTROL),
		   sdhci_readl(host, SDHCI_INT_STATUS));
	SDHCI_DUMP("Int enab:  0x%08x | Sig enab: 0x%08x\n",
		   sdhci_readl(host, SDHCI_INT_ENABLE),
		   sdhci_readl(host, SDHCI_SIGNAL_ENABLE));
	SDHCI_DUMP("AC12 err:  0x%08x | Slot int: 0x%08x\n",
		   sdhci_readw(host, SDHCI_ACMD12_ERR),
		   sdhci_readw(host, SDHCI_SLOT_INT_STATUS));
	SDHCI_DUMP("Caps:      0x%08x | Caps_1:   0x%08x\n",
		   sdhci_readl(host, SDHCI_CAPABILITIES),
		   sdhci_readl(host, SDHCI_CAPABILITIES_1));
	SDHCI_DUMP("Cmd:       0x%08x | Max curr: 0x%08x\n",
		   sdhci_readw(host, SDHCI_COMMAND),
		   sdhci_readl(host, SDHCI_MAX_CURRENT));
	SDHCI_DUMP("Resp[0]:   0x%08x | Resp[1]:  0x%08x\n",
95
96
		   sdhci_readl(host, SDHCI_RESPONSE),
		   sdhci_readl(host, SDHCI_RESPONSE + 4));
97
	SDHCI_DUMP("Resp[2]:   0x%08x | Resp[3]:  0x%08x\n",
98
99
		   sdhci_readl(host, SDHCI_RESPONSE + 8),
		   sdhci_readl(host, SDHCI_RESPONSE + 12));
100
101
	SDHCI_DUMP("Host ctl2: 0x%08x\n",
		   sdhci_readw(host, SDHCI_HOST_CONTROL2));
102

103
	if (host->flags & SDHCI_USE_ADMA) {
104
105
106
107
108
109
110
111
112
113
		if (host->flags & SDHCI_USE_64_BIT_DMA) {
			SDHCI_DUMP("ADMA Err:  0x%08x | ADMA Ptr: 0x%08x%08x\n",
				   sdhci_readl(host, SDHCI_ADMA_ERROR),
				   sdhci_readl(host, SDHCI_ADMA_ADDRESS_HI),
				   sdhci_readl(host, SDHCI_ADMA_ADDRESS));
		} else {
			SDHCI_DUMP("ADMA Err:  0x%08x | ADMA Ptr: 0x%08x\n",
				   sdhci_readl(host, SDHCI_ADMA_ERROR),
				   sdhci_readl(host, SDHCI_ADMA_ADDRESS));
		}
114
	}
115

116
	SDHCI_DUMP("============================================\n");
117
}
118
EXPORT_SYMBOL_GPL(sdhci_dumpregs);
119
120
121
122
123
124
125

/*****************************************************************************\
 *                                                                           *
 * Low level functions                                                       *
 *                                                                           *
\*****************************************************************************/

126
127
128
129
130
static inline bool sdhci_data_line_cmd(struct mmc_command *cmd)
{
	return cmd->data || cmd->flags & MMC_RSP_BUSY;
}

131
132
static void sdhci_set_card_detection(struct sdhci_host *host, bool enable)
{
133
	u32 present;
134

135
	if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) ||
136
	    !mmc_card_is_removable(host->mmc))
137
138
		return;

139
140
141
	if (enable) {
		present = sdhci_readl(host, SDHCI_PRESENT_STATE) &
				      SDHCI_CARD_PRESENT;
142

143
144
145
146
147
		host->ier |= present ? SDHCI_INT_CARD_REMOVE :
				       SDHCI_INT_CARD_INSERT;
	} else {
		host->ier &= ~(SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT);
	}
148
149
150

	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
151
152
153
154
155
156
157
158
159
160
161
162
}

static void sdhci_enable_card_detection(struct sdhci_host *host)
{
	sdhci_set_card_detection(host, true);
}

static void sdhci_disable_card_detection(struct sdhci_host *host)
{
	sdhci_set_card_detection(host, false);
}

163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
static void sdhci_runtime_pm_bus_on(struct sdhci_host *host)
{
	if (host->bus_on)
		return;
	host->bus_on = true;
	pm_runtime_get_noresume(host->mmc->parent);
}

static void sdhci_runtime_pm_bus_off(struct sdhci_host *host)
{
	if (!host->bus_on)
		return;
	host->bus_on = false;
	pm_runtime_put_noidle(host->mmc->parent);
}

179
void sdhci_reset(struct sdhci_host *host, u8 mask)
180
{
181
	ktime_t timeout;
182

183
	sdhci_writeb(host, mask, SDHCI_SOFTWARE_RESET);
184

185
	if (mask & SDHCI_RESET_ALL) {
186
		host->clock = 0;
187
188
189
190
		/* Reset-all turns off SD Bus Power */
		if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
			sdhci_runtime_pm_bus_off(host);
	}
191

192
	/* Wait max 100 ms */
193
	timeout = ktime_add_ms(ktime_get(), 100);
194
195

	/* hw clears the bit when it's done */
196
197
198
199
200
201
	while (1) {
		bool timedout = ktime_after(ktime_get(), timeout);

		if (!(sdhci_readb(host, SDHCI_SOFTWARE_RESET) & mask))
			break;
		if (timedout) {
202
			pr_err("%s: Reset 0x%x never completed.\n",
203
204
205
206
				mmc_hostname(host->mmc), (int)mask);
			sdhci_dumpregs(host);
			return;
		}
207
		udelay(10);
208
	}
209
210
211
212
213
214
}
EXPORT_SYMBOL_GPL(sdhci_reset);

static void sdhci_do_reset(struct sdhci_host *host, u8 mask)
{
	if (host->quirks & SDHCI_QUIRK_NO_CARD_NO_RESET) {
215
216
217
		struct mmc_host *mmc = host->mmc;

		if (!mmc->ops->get_cd(mmc))
218
219
			return;
	}
220

221
	host->ops->reset(host, mask);
222

223
224
225
226
227
228
229
230
	if (mask & SDHCI_RESET_ALL) {
		if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
			if (host->ops->enable_dma)
				host->ops->enable_dma(host);
		}

		/* Resetting the controller clears many */
		host->preset_enabled = false;
231
	}
232
233
}

234
static void sdhci_set_default_irqs(struct sdhci_host *host)
235
{
236
237
238
239
240
241
	host->ier = SDHCI_INT_BUS_POWER | SDHCI_INT_DATA_END_BIT |
		    SDHCI_INT_DATA_CRC | SDHCI_INT_DATA_TIMEOUT |
		    SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC |
		    SDHCI_INT_TIMEOUT | SDHCI_INT_DATA_END |
		    SDHCI_INT_RESPONSE;

242
243
244
245
	if (host->tuning_mode == SDHCI_TUNING_MODE_2 ||
	    host->tuning_mode == SDHCI_TUNING_MODE_3)
		host->ier |= SDHCI_INT_RETUNE;

246
247
	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
248
249
250
251
252
253
254
255
256
257
258
259
}

static void sdhci_init(struct sdhci_host *host, int soft)
{
	struct mmc_host *mmc = host->mmc;

	if (soft)
		sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
	else
		sdhci_do_reset(host, SDHCI_RESET_ALL);

	sdhci_set_default_irqs(host);
260

Adrian Hunter's avatar
Adrian Hunter committed
261
262
	host->cqe_on = false;

263
264
265
	if (soft) {
		/* force clock reconfiguration */
		host->clock = 0;
266
		mmc->ops->set_ios(mmc, &mmc->ios);
267
	}
268
}
269

270
271
static void sdhci_reinit(struct sdhci_host *host)
{
272
	sdhci_init(host, 0);
273
	sdhci_enable_card_detection(host);
274
275
}

276
static void __sdhci_led_activate(struct sdhci_host *host)
277
278
279
{
	u8 ctrl;

280
	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
281
	ctrl |= SDHCI_CTRL_LED;
282
	sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
283
284
}

285
static void __sdhci_led_deactivate(struct sdhci_host *host)
286
287
288
{
	u8 ctrl;

289
	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
290
	ctrl &= ~SDHCI_CTRL_LED;
291
	sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
292
293
}

294
#if IS_REACHABLE(CONFIG_LEDS_CLASS)
295
static void sdhci_led_control(struct led_classdev *led,
296
			      enum led_brightness brightness)
297
298
299
300
301
302
{
	struct sdhci_host *host = container_of(led, struct sdhci_host, led);
	unsigned long flags;

	spin_lock_irqsave(&host->lock, flags);

303
304
305
	if (host->runtime_suspended)
		goto out;

306
	if (brightness == LED_OFF)
307
		__sdhci_led_deactivate(host);
308
	else
309
		__sdhci_led_activate(host);
310
out:
311
312
	spin_unlock_irqrestore(&host->lock, flags);
}
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362

static int sdhci_led_register(struct sdhci_host *host)
{
	struct mmc_host *mmc = host->mmc;

	snprintf(host->led_name, sizeof(host->led_name),
		 "%s::", mmc_hostname(mmc));

	host->led.name = host->led_name;
	host->led.brightness = LED_OFF;
	host->led.default_trigger = mmc_hostname(mmc);
	host->led.brightness_set = sdhci_led_control;

	return led_classdev_register(mmc_dev(mmc), &host->led);
}

static void sdhci_led_unregister(struct sdhci_host *host)
{
	led_classdev_unregister(&host->led);
}

static inline void sdhci_led_activate(struct sdhci_host *host)
{
}

static inline void sdhci_led_deactivate(struct sdhci_host *host)
{
}

#else

static inline int sdhci_led_register(struct sdhci_host *host)
{
	return 0;
}

static inline void sdhci_led_unregister(struct sdhci_host *host)
{
}

static inline void sdhci_led_activate(struct sdhci_host *host)
{
	__sdhci_led_activate(host);
}

static inline void sdhci_led_deactivate(struct sdhci_host *host)
{
	__sdhci_led_deactivate(host);
}

363
364
#endif

365
366
367
368
369
370
/*****************************************************************************\
 *                                                                           *
 * Core functions                                                            *
 *                                                                           *
\*****************************************************************************/

Pierre Ossman's avatar
Pierre Ossman committed
371
static void sdhci_read_block_pio(struct sdhci_host *host)
372
{
373
374
	unsigned long flags;
	size_t blksize, len, chunk;
375
	u32 uninitialized_var(scratch);
376
	u8 *buf;
377

Pierre Ossman's avatar
Pierre Ossman committed
378
	DBG("PIO reading\n");
379

Pierre Ossman's avatar
Pierre Ossman committed
380
	blksize = host->data->blksz;
381
	chunk = 0;
382

383
	local_irq_save(flags);
384

Pierre Ossman's avatar
Pierre Ossman committed
385
	while (blksize) {
Fabio Estevam's avatar
Fabio Estevam committed
386
		BUG_ON(!sg_miter_next(&host->sg_miter));
387

388
		len = min(host->sg_miter.length, blksize);
389

390
391
		blksize -= len;
		host->sg_miter.consumed = len;
392

393
		buf = host->sg_miter.addr;
394

395
396
		while (len) {
			if (chunk == 0) {
397
				scratch = sdhci_readl(host, SDHCI_BUFFER);
398
				chunk = 4;
Pierre Ossman's avatar
Pierre Ossman committed
399
			}
400
401
402
403
404
405
406

			*buf = scratch & 0xFF;

			buf++;
			scratch >>= 8;
			chunk--;
			len--;
407
		}
Pierre Ossman's avatar
Pierre Ossman committed
408
	}
409
410
411
412

	sg_miter_stop(&host->sg_miter);

	local_irq_restore(flags);
Pierre Ossman's avatar
Pierre Ossman committed
413
}
414

Pierre Ossman's avatar
Pierre Ossman committed
415
416
static void sdhci_write_block_pio(struct sdhci_host *host)
{
417
418
419
420
	unsigned long flags;
	size_t blksize, len, chunk;
	u32 scratch;
	u8 *buf;
421

Pierre Ossman's avatar
Pierre Ossman committed
422
423
424
	DBG("PIO writing\n");

	blksize = host->data->blksz;
425
426
	chunk = 0;
	scratch = 0;
427

428
	local_irq_save(flags);
429

Pierre Ossman's avatar
Pierre Ossman committed
430
	while (blksize) {
Fabio Estevam's avatar
Fabio Estevam committed
431
		BUG_ON(!sg_miter_next(&host->sg_miter));
Pierre Ossman's avatar
Pierre Ossman committed
432

433
434
435
436
437
438
		len = min(host->sg_miter.length, blksize);

		blksize -= len;
		host->sg_miter.consumed = len;

		buf = host->sg_miter.addr;
439

440
441
442
443
444
445
446
447
		while (len) {
			scratch |= (u32)*buf << (chunk * 8);

			buf++;
			chunk++;
			len--;

			if ((chunk == 4) || ((len == 0) && (blksize == 0))) {
448
				sdhci_writel(host, scratch, SDHCI_BUFFER);
449
450
				chunk = 0;
				scratch = 0;
451
452
453
			}
		}
	}
454
455
456
457

	sg_miter_stop(&host->sg_miter);

	local_irq_restore(flags);
Pierre Ossman's avatar
Pierre Ossman committed
458
459
460
461
462
463
}

static void sdhci_transfer_pio(struct sdhci_host *host)
{
	u32 mask;

464
	if (host->blocks == 0)
Pierre Ossman's avatar
Pierre Ossman committed
465
466
467
468
469
470
471
		return;

	if (host->data->flags & MMC_DATA_READ)
		mask = SDHCI_DATA_AVAILABLE;
	else
		mask = SDHCI_SPACE_AVAILABLE;

472
473
474
475
476
477
478
479
480
	/*
	 * Some controllers (JMicron JMB38x) mess up the buffer bits
	 * for transfers < 4 bytes. As long as it is just one block,
	 * we can ignore the bits.
	 */
	if ((host->quirks & SDHCI_QUIRK_BROKEN_SMALL_PIO) &&
		(host->data->blocks == 1))
		mask = ~0;

481
	while (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask) {
482
483
484
		if (host->quirks & SDHCI_QUIRK_PIO_NEEDS_DELAY)
			udelay(100);

Pierre Ossman's avatar
Pierre Ossman committed
485
486
487
488
		if (host->data->flags & MMC_DATA_READ)
			sdhci_read_block_pio(host);
		else
			sdhci_write_block_pio(host);
489

490
491
		host->blocks--;
		if (host->blocks == 0)
Pierre Ossman's avatar
Pierre Ossman committed
492
493
			break;
	}
494

Pierre Ossman's avatar
Pierre Ossman committed
495
	DBG("PIO transfer complete.\n");
496
497
}

498
static int sdhci_pre_dma_transfer(struct sdhci_host *host,
499
				  struct mmc_data *data, int cookie)
500
501
502
{
	int sg_count;

503
504
505
506
507
	/*
	 * If the data buffers are already mapped, return the previous
	 * dma_map_sg() result.
	 */
	if (data->host_cookie == COOKIE_PRE_MAPPED)
508
509
		return data->sg_count;

510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
	/* Bounce write requests to the bounce buffer */
	if (host->bounce_buffer) {
		unsigned int length = data->blksz * data->blocks;

		if (length > host->bounce_buffer_size) {
			pr_err("%s: asked for transfer of %u bytes exceeds bounce buffer %u bytes\n",
			       mmc_hostname(host->mmc), length,
			       host->bounce_buffer_size);
			return -EIO;
		}
		if (mmc_get_dma_dir(data) == DMA_TO_DEVICE) {
			/* Copy the data to the bounce buffer */
			sg_copy_to_buffer(data->sg, data->sg_len,
					  host->bounce_buffer,
					  length);
		}
		/* Switch ownership to the DMA */
		dma_sync_single_for_device(host->mmc->parent,
					   host->bounce_addr,
					   host->bounce_buffer_size,
					   mmc_get_dma_dir(data));
		/* Just a dummy value */
		sg_count = 1;
	} else {
		/* Just access the data directly from memory */
		sg_count = dma_map_sg(mmc_dev(host->mmc),
				      data->sg, data->sg_len,
				      mmc_get_dma_dir(data));
	}
539
540
541
542
543

	if (sg_count == 0)
		return -ENOSPC;

	data->sg_count = sg_count;
544
	data->host_cookie = cookie;
545
546
547
548

	return sg_count;
}

549
550
551
static char *sdhci_kmap_atomic(struct scatterlist *sg, unsigned long *flags)
{
	local_irq_save(*flags);
552
	return kmap_atomic(sg_page(sg)) + sg->offset;
553
554
555
556
}

static void sdhci_kunmap_atomic(void *buffer, unsigned long *flags)
{
557
	kunmap_atomic(buffer);
558
559
560
	local_irq_restore(*flags);
}

561
562
static void sdhci_adma_write_desc(struct sdhci_host *host, void *desc,
				  dma_addr_t addr, int len, unsigned cmd)
563
{
564
	struct sdhci_adma2_64_desc *dma_desc = desc;
565

566
	/* 32-bit and 64-bit descriptors have these members in same position */
567
568
	dma_desc->cmd = cpu_to_le16(cmd);
	dma_desc->len = cpu_to_le16(len);
569
570
571
572
	dma_desc->addr_lo = cpu_to_le32((u32)addr);

	if (host->flags & SDHCI_USE_64_BIT_DMA)
		dma_desc->addr_hi = cpu_to_le32((u64)addr >> 32);
573
574
}

575
576
static void sdhci_adma_mark_end(void *desc)
{
577
	struct sdhci_adma2_64_desc *dma_desc = desc;
578

579
	/* 32-bit and 64-bit descriptors have 'cmd' in same position */
580
	dma_desc->cmd |= cpu_to_le16(ADMA2_END);
581
582
}

583
584
static void sdhci_adma_table_pre(struct sdhci_host *host,
	struct mmc_data *data, int sg_count)
585
586
587
{
	struct scatterlist *sg;
	unsigned long flags;
588
589
590
591
	dma_addr_t addr, align_addr;
	void *desc, *align;
	char *buffer;
	int len, offset, i;
592
593
594
595
596
597

	/*
	 * The spec does not specify endianness of descriptor table.
	 * We currently guess that it is LE.
	 */

598
	host->sg_count = sg_count;
599

600
	desc = host->adma_table;
601
602
603
604
605
606
607
608
609
	align = host->align_buffer;

	align_addr = host->align_addr;

	for_each_sg(data->sg, sg, host->sg_count, i) {
		addr = sg_dma_address(sg);
		len = sg_dma_len(sg);

		/*
610
611
612
		 * The SDHCI specification states that ADMA addresses must
		 * be 32-bit aligned. If they aren't, then we use a bounce
		 * buffer for the (up to three) bytes that screw up the
613
614
		 * alignment.
		 */
615
616
		offset = (SDHCI_ADMA2_ALIGN - (addr & SDHCI_ADMA2_MASK)) &
			 SDHCI_ADMA2_MASK;
617
618
619
620
621
622
623
		if (offset) {
			if (data->flags & MMC_DATA_WRITE) {
				buffer = sdhci_kmap_atomic(sg, &flags);
				memcpy(align, buffer, offset);
				sdhci_kunmap_atomic(buffer, &flags);
			}

624
			/* tran, valid */
625
			sdhci_adma_write_desc(host, desc, align_addr, offset,
626
					      ADMA2_TRAN_VALID);
627
628
629

			BUG_ON(offset > 65536);

630
631
			align += SDHCI_ADMA2_ALIGN;
			align_addr += SDHCI_ADMA2_ALIGN;
632

633
			desc += host->desc_sz;
634
635
636
637
638
639
640

			addr += offset;
			len -= offset;
		}

		BUG_ON(len > 65536);

641
642
643
644
645
646
		if (len) {
			/* tran, valid */
			sdhci_adma_write_desc(host, desc, addr, len,
					      ADMA2_TRAN_VALID);
			desc += host->desc_sz;
		}
647
648
649
650
651

		/*
		 * If this triggers then we have a calculation bug
		 * somewhere. :/
		 */
652
		WARN_ON((desc - host->adma_table) >= host->adma_table_sz);
653
654
	}

655
	if (host->quirks & SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC) {
656
		/* Mark the last descriptor as the terminating descriptor */
657
		if (desc != host->adma_table) {
658
			desc -= host->desc_sz;
659
			sdhci_adma_mark_end(desc);
660
661
		}
	} else {
662
		/* Add a terminating entry - nop, end, valid */
663
		sdhci_adma_write_desc(host, desc, 0, 0, ADMA2_NOP_END_VALID);
664
	}
665
666
667
668
669
670
671
}

static void sdhci_adma_table_post(struct sdhci_host *host,
	struct mmc_data *data)
{
	struct scatterlist *sg;
	int i, size;
672
	void *align;
673
674
675
	char *buffer;
	unsigned long flags;

676
677
	if (data->flags & MMC_DATA_READ) {
		bool has_unaligned = false;
678

679
680
681
682
683
684
		/* Do a quick scan of the SG list for any unaligned mappings */
		for_each_sg(data->sg, sg, host->sg_count, i)
			if (sg_dma_address(sg) & SDHCI_ADMA2_MASK) {
				has_unaligned = true;
				break;
			}
685

686
687
		if (has_unaligned) {
			dma_sync_sg_for_cpu(mmc_dev(host->mmc), data->sg,
688
					    data->sg_len, DMA_FROM_DEVICE);
689

690
			align = host->align_buffer;
691

692
693
694
695
696
697
698
699
			for_each_sg(data->sg, sg, host->sg_count, i) {
				if (sg_dma_address(sg) & SDHCI_ADMA2_MASK) {
					size = SDHCI_ADMA2_ALIGN -
					       (sg_dma_address(sg) & SDHCI_ADMA2_MASK);

					buffer = sdhci_kmap_atomic(sg, &flags);
					memcpy(buffer, align, size);
					sdhci_kunmap_atomic(buffer, &flags);
700

701
702
					align += SDHCI_ADMA2_ALIGN;
				}
703
704
705
706
707
			}
		}
	}
}

708
709
710
711
712
713
714
715
static u32 sdhci_sdma_address(struct sdhci_host *host)
{
	if (host->bounce_buffer)
		return host->bounce_addr;
	else
		return sg_dma_address(host->data->sg);
}

716
static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd)
717
{
718
	u8 count;
719
	struct mmc_data *data = cmd->data;
720
	unsigned target_timeout, current_timeout;
721

722
723
724
725
726
727
	/*
	 * If the host controller provides us with an incorrect timeout
	 * value, just skip the check and use 0xE.  The hardware may take
	 * longer to time out, but that's much better than having a too-short
	 * timeout value.
	 */
728
	if (host->quirks & SDHCI_QUIRK_BROKEN_TIMEOUT_VAL)
729
		return 0xE;
730

731
	/* Unspecified timeout, assume max */
732
	if (!data && !cmd->busy_timeout)
733
		return 0xE;
734

735
736
	/* timeout in us */
	if (!data)
737
		target_timeout = cmd->busy_timeout * 1000;
738
	else {
739
		target_timeout = DIV_ROUND_UP(data->timeout_ns, 1000);
740
741
742
743
744
745
746
747
		if (host->clock && data->timeout_clks) {
			unsigned long long val;

			/*
			 * data->timeout_clks is in units of clock cycles.
			 * host->clock is in Hz.  target_timeout is in us.
			 * Hence, us = 1000000 * cycles / Hz.  Round up.
			 */
748
			val = 1000000ULL * data->timeout_clks;
749
750
751
752
			if (do_div(val, host->clock))
				target_timeout++;
			target_timeout += val;
		}
753
	}
754

755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
	/*
	 * Figure out needed cycles.
	 * We do this in steps in order to fit inside a 32 bit int.
	 * The first step is the minimum timeout, which will have a
	 * minimum resolution of 6 bits:
	 * (1) 2^13*1000 > 2^22,
	 * (2) host->timeout_clk < 2^16
	 *     =>
	 *     (1) / (2) > 2^6
	 */
	count = 0;
	current_timeout = (1 << 13) * 1000 / host->timeout_clk;
	while (current_timeout < target_timeout) {
		count++;
		current_timeout <<= 1;
		if (count >= 0xF)
			break;
	}

	if (count >= 0xF) {
775
776
		DBG("Too large timeout 0x%x requested for CMD%d!\n",
		    count, cmd->opcode);
777
778
779
		count = 0xE;
	}

780
781
782
	return count;
}

783
784
785
786
787
788
static void sdhci_set_transfer_irqs(struct sdhci_host *host)
{
	u32 pio_irqs = SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL;
	u32 dma_irqs = SDHCI_INT_DMA_END | SDHCI_INT_ADMA_ERROR;

	if (host->flags & SDHCI_REQ_USE_DMA)
789
		host->ier = (host->ier & ~pio_irqs) | dma_irqs;
790
	else
791
792
793
794
		host->ier = (host->ier & ~dma_irqs) | pio_irqs;

	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
795
796
}

797
static void sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd)
798
799
{
	u8 count;
800
801
802
803
804
805
806
807
808
809
810

	if (host->ops->set_timeout) {
		host->ops->set_timeout(host, cmd);
	} else {
		count = sdhci_calc_timeout(host, cmd);
		sdhci_writeb(host, count, SDHCI_TIMEOUT_CONTROL);
	}
}

static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd)
{
811
	u8 ctrl;
812
	struct mmc_data *data = cmd->data;
813

814
	if (sdhci_data_line_cmd(cmd))
815
		sdhci_set_timeout(host, cmd);
816
817

	if (!data)
818
819
		return;

820
821
	WARN_ON(host->data);

822
823
824
825
826
827
828
	/* Sanity checks */
	BUG_ON(data->blksz * data->blocks > 524288);
	BUG_ON(data->blksz > host->mmc->max_blk_size);
	BUG_ON(data->blocks > 65535);

	host->data = data;
	host->data_early = 0;
829
	host->data->bytes_xfered = 0;
830

831
	if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
832
		struct scatterlist *sg;
833
		unsigned int length_mask, offset_mask;
834
		int i;
835

836
837
838
839
840
841
842
843
844
		host->flags |= SDHCI_REQ_USE_DMA;

		/*
		 * FIXME: This doesn't account for merging when mapping the
		 * scatterlist.
		 *
		 * The assumption here being that alignment and lengths are
		 * the same after DMA mapping to device address space.
		 */
845
		length_mask = 0;
846
		offset_mask = 0;
847
		if (host->flags & SDHCI_USE_ADMA) {
848
			if (host->quirks & SDHCI_QUIRK_32BIT_ADMA_SIZE) {
849
				length_mask = 3;
850
851
852
853
854
855
856
				/*
				 * As we use up to 3 byte chunks to work
				 * around alignment problems, we need to
				 * check the offset as well.
				 */
				offset_mask = 3;
			}
857
858
		} else {
			if (host->quirks & SDHCI_QUIRK_32BIT_DMA_SIZE)
859
				length_mask = 3;
860
861
			if (host->quirks & SDHCI_QUIRK_32BIT_DMA_ADDR)
				offset_mask = 3;
862
863
		}

864
		if (unlikely(length_mask | offset_mask)) {
865
			for_each_sg(data->sg, sg, data->sg_len, i) {
866
				if (sg->length & length_mask) {
867
					DBG("Reverting to PIO because of transfer size (%d)\n",
868
					    sg->length);
869
870
871
					host->flags &= ~SDHCI_REQ_USE_DMA;
					break;
				}
872
				if (sg->offset & offset_mask) {
873
					DBG("Reverting to PIO because of bad alignment\n");
874
875
876
877
878
879
880
					host->flags &= ~SDHCI_REQ_USE_DMA;
					break;
				}
			}
		}
	}

881
	if (host->flags & SDHCI_REQ_USE_DMA) {
882
		int sg_cnt = sdhci_pre_dma_transfer(host, data, COOKIE_MAPPED);
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898

		if (sg_cnt <= 0) {
			/*
			 * This only happens when someone fed
			 * us an invalid request.
			 */
			WARN_ON(1);
			host->flags &= ~SDHCI_REQ_USE_DMA;
		} else if (host->flags & SDHCI_USE_ADMA) {
			sdhci_adma_table_pre(host, data, sg_cnt);

			sdhci_writel(host, host->adma_addr, SDHCI_ADMA_ADDRESS);
			if (host->flags & SDHCI_USE_64_BIT_DMA)
				sdhci_writel(host,
					     (u64)host->adma_addr >> 32,
					     SDHCI_ADMA_ADDRESS_HI);
899
		} else {
900
			WARN_ON(sg_cnt != 1);
901
902
			sdhci_writel(host, sdhci_sdma_address(host),
				     SDHCI_DMA_ADDRESS);
903
904
905
		}
	}

906
907
908
909
910
911
	/*
	 * Always adjust the DMA selection as some controllers
	 * (e.g. JMicron) can't do PIO properly when the selection
	 * is ADMA.
	 */
	if (host->version >= SDHCI_SPEC_200) {
912
		ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
913
914
		ctrl &= ~SDHCI_CTRL_DMA_MASK;
		if ((host->flags & SDHCI_REQ_USE_DMA) &&
915
916
917
918
919
920
			(host->flags & SDHCI_USE_ADMA)) {
			if (host->flags & SDHCI_USE_64_BIT_DMA)
				ctrl |= SDHCI_CTRL_ADMA64;
			else
				ctrl |= SDHCI_CTRL_ADMA32;
		} else {
921
			ctrl |= SDHCI_CTRL_SDMA;
922
		}
923
		sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
924
925
	}

926
	if (!(host->flags & SDHCI_REQ_USE_DMA)) {
927
928
929
930
931
932
933
934
		int flags;

		flags = SG_MITER_ATOMIC;
		if (host->data->flags & MMC_DATA_READ)
			flags |= SG_MITER_TO_SG;
		else
			flags |= SG_MITER_FROM_SG;
		sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);
935
		host->blocks = data->blocks;
936
	}
937

938
939
	sdhci_set_transfer_irqs(host);

940
	/* Set the DMA boundary value and block size */
941
942
	sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, data->blksz),
		     SDHCI_BLOCK_SIZE);
943
	sdhci_writew(host, data->blocks, SDHCI_BLOCK_COUNT);
944
945
}

946
947
948
static inline bool sdhci_auto_cmd12(struct sdhci_host *host,
				    struct mmc_request *mrq)
{
949
950
	return !mrq->sbc && (host->flags & SDHCI_AUTO_CMD12) &&
	       !mrq->cap_cmd_during_tfr;
951
952
}

953
static void sdhci_set_transfer_mode(struct sdhci_host *host,
954
	struct mmc_command *cmd)
955
{
956
	u16 mode = 0;
957
	struct mmc_data *data = cmd->data;
958

959
	if (data == NULL) {
960
961
962
963
		if (host->quirks2 &
			SDHCI_QUIRK2_CLEAR_TRANSFERMODE_REG_BEFORE_CMD) {
			sdhci_writew(host, 0x0, SDHCI_TRANSFER_MODE);
		} else {
964
		/* clear Auto CMD settings for no data CMDs */
965
966
			mode = sdhci_readw(host, SDHCI_TRANSFER_MODE);
			sdhci_writew(host, mode & ~(SDHCI_TRNS_AUTO_CMD12 |
967
				SDHCI_TRNS_AUTO_CMD23), SDHCI_TRANSFER_MODE);
968
		}
969
		return;
970
	}
971

972
973
	WARN_ON(!host->data);

974
975
976
	if (!(host->quirks2 & SDHCI_QUIRK2_SUPPORT_SINGLE))
		mode = SDHCI_TRNS_BLK_CNT_EN;

977
	if (mmc_op_multi(cmd->opcode) || data->blocks > 1) {
978
		mode = SDHCI_TRNS_BLK_CNT_EN | SDHCI_TRNS_MULTI;
979
980
981
982
		/*
		 * If we are sending CMD23, CMD12 never gets sent
		 * on successful completion (so no Auto-CMD12).
		 */
983
		if (sdhci_auto_cmd12(host, cmd->mrq) &&
984
		    (cmd->opcode != SD_IO_RW_EXTENDED))
985
			mode |= SDHCI_TRNS_AUTO_CMD12;
986
		else if (cmd->mrq->sbc && (host->flags & SDHCI_AUTO_CMD23)) {
987
			mode |= SDHCI_TRNS_AUTO_CMD23;
988
			sdhci_writel(host, cmd->mrq->sbc->arg, SDHCI_ARGUMENT2);
989
		}
990
	}
991

992
993
	if (data->flags & MMC_DATA_READ)
		mode |= SDHCI_TRNS_READ;
994
	if (host->flags & SDHCI_REQ_USE_DMA)
995
996
		mode |= SDHCI_TRNS_DMA;

997
	sdhci_writew(host, mode, SDHCI_TRANSFER_MODE);
998
999
}

1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
static bool sdhci_needs_reset(struct sdhci_host *host, struct mmc_request *mrq)
{
	return (!(host->flags & SDHCI_DEVICE_DEAD) &&
		((mrq->cmd && mrq->cmd->error) ||
		 (mrq->sbc && mrq->sbc->error) ||
		 (mrq->data && ((mrq->data->error && !mrq->data->stop) ||
				(mrq->data->stop && mrq->data->stop->error))) ||
		 (host->quirks & SDHCI_QUIRK_RESET_AFTER_REQUEST)));
}

1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
static void __sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq)
{
	int i;

	for (i = 0; i < SDHCI_MAX_MRQS; i++) {
		if (host->mrqs_done[i] == mrq) {
			WARN_ON(1);
			return;
		}
	}

	for (i = 0; i < SDHCI_MAX_MRQS; i++) {
		if (!host->mrqs_done[i]) {
			host->mrqs_done[i] = mrq;
			break;
		}
	}

	WARN_ON(i >= SDHCI_MAX_MRQS);

	tasklet_schedule(&host->finish_tasklet);
}

1033
1034
static void sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq)
{
1035
1036
1037
1038
1039
1040
1041
1042
1043
	if (host->cmd && host->cmd->mrq == mrq)
		host->cmd = NULL;

	if (host->data_cmd && host->data_cmd->mrq == mrq)
		host->data_cmd = NULL;

	if (host->data && host->data->mrq == mrq)
		host->data = NULL;

1044
1045
1046
	if (sdhci_needs_reset(host, mrq))
		host->pending_reset = true;

1047
	__sdhci_finish_mrq(host, mrq);
1048
1049
}

1050
1051
static void sdhci_finish_data(struct sdhci_host *host)
{
1052
1053
	struct mmc_command *data_cmd = host->data_cmd;
	struct mmc_data *data = host->data;
1054
1055

	host->data = NULL;
1056
	host->data_cmd = NULL;
1057

1058
1059
1060
	if ((host->flags & (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA)) ==
	    (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA))
		sdhci_adma_table_post(host, data);
1061
1062

	/*
1063
1064
1065
1066
1067
	 * The specification states that the block count register must
	 * be updated, but it does not specify at what point in the
	 * data flow. That makes the register entirely useless to read
	 * back so we have to assume that nothing made it to the card
	 * in the event of an error.
1068
	 */
1069
1070
	if (data->error)
		data->bytes_xfered = 0;
1071
	else
1072
		data->bytes_xfered = data->blksz * data->blocks;
1073

1074
1075
1076
1077
1078
1079
1080
	/*
	 * Need to send CMD12 if -
	 * a) open-ended multiblock transfer (no CMD23)
	 * b) error in multiblock transfer
	 */
	if (data->stop &&
	    (data->error ||
1081
	     !data->mrq->sbc)) {
1082

1083
1084
1085
1086
		/*
		 * The controller needs a reset of internal state machines
		 * upon error conditions.
		 */
Pierre Ossman's avatar
Pierre Ossman committed
1087
		if (data->error) {
1088
1089
			if (!host->cmd || host->cmd == data_cmd)
				sdhci_do_reset(host, SDHCI_RESET_CMD);
1090
			sdhci_do_reset(host, SDHCI_RESET_DATA);
1091
1092
		}