pci.c 24.2 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
/*
 * Support PCI/PCIe on PowerNV platforms
 *
 * Currently supports only P5IOC2
 *
 * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/delay.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/irq.h>
#include <linux/io.h>
22
#include <linux/msi.h>
23
#include <linux/iommu.h>
24
25
26
27
28
29

#include <asm/sections.h>
#include <asm/io.h>
#include <asm/prom.h>
#include <asm/pci-bridge.h>
#include <asm/machdep.h>
30
#include <asm/msi_bitmap.h>
31
32
33
34
#include <asm/ppc-pci.h>
#include <asm/opal.h>
#include <asm/iommu.h>
#include <asm/tce.h>
35
#include <asm/firmware.h>
36
37
#include <asm/eeh_event.h>
#include <asm/eeh.h>
38
39
40
41

#include "powernv.h"
#include "pci.h"

42
43
/* Delay in usec */
#define PCI_RESET_DELAY_US	3000000
44
45
46
47

#define cfg_dbg(fmt...)	do { } while(0)
//#define cfg_dbg(fmt...)	printk(fmt)

48
49
50
51
52
#ifdef CONFIG_PCI_MSI
static int pnv_msi_check_device(struct pci_dev* pdev, int nvec, int type)
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;
53
54
55
56
	struct pci_dn *pdn = pci_get_pdn(pdev);

	if (pdn && pdn->force_32bit_msi && !phb->msi32_support)
		return -ENODEV;
57

58
	return (phb && phb->msi_bmp.bitmap) ? 0 : -ENODEV;
59
60
61
62
63
64
65
66
}

static int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;
	struct msi_desc *entry;
	struct msi_msg msg;
67
68
	int hwirq;
	unsigned int virq;
69
70
71
72
73
74
75
76
77
78
79
	int rc;

	if (WARN_ON(!phb))
		return -ENODEV;

	list_for_each_entry(entry, &pdev->msi_list, list) {
		if (!entry->msi_attrib.is_64 && !phb->msi32_support) {
			pr_warn("%s: Supports only 64-bit MSIs\n",
				pci_name(pdev));
			return -ENXIO;
		}
80
81
		hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, 1);
		if (hwirq < 0) {
82
83
84
85
			pr_warn("%s: Failed to find a free MSI\n",
				pci_name(pdev));
			return -ENOSPC;
		}
86
		virq = irq_create_mapping(NULL, phb->msi_base + hwirq);
87
88
89
		if (virq == NO_IRQ) {
			pr_warn("%s: Failed to map MSI to linux irq\n",
				pci_name(pdev));
90
			msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
91
92
			return -ENOMEM;
		}
93
		rc = phb->msi_setup(phb, pdev, phb->msi_base + hwirq,
94
				    virq, entry->msi_attrib.is_64, &msg);
95
96
97
		if (rc) {
			pr_warn("%s: Failed to setup MSI\n", pci_name(pdev));
			irq_dispose_mapping(virq);
98
			msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
			return rc;
		}
		irq_set_msi_desc(virq, entry);
		write_msi_msg(virq, &msg);
	}
	return 0;
}

static void pnv_teardown_msi_irqs(struct pci_dev *pdev)
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;
	struct msi_desc *entry;

	if (WARN_ON(!phb))
		return;

	list_for_each_entry(entry, &pdev->msi_list, list) {
		if (entry->irq == NO_IRQ)
			continue;
		irq_set_msi_desc(entry->irq, NULL);
120
121
		msi_bitmap_free_hwirqs(&phb->msi_bmp,
			virq_to_hw(entry->irq) - phb->msi_base, 1);
122
123
124
125
		irq_dispose_mapping(entry->irq);
	}
}
#endif /* CONFIG_PCI_MSI */
126

127
128
static void pnv_pci_dump_p7ioc_diag_data(struct pci_controller *hose,
					 struct OpalIoPhbErrorCommon *common)
129
{
130
	struct OpalIoP7IOCPhbErrorData *data;
131
132
	int i;

133
	data = (struct OpalIoP7IOCPhbErrorData *)common;
134
	pr_info("P7IOC PHB#%d Diag-data (Version: %d)\n",
135
		hose->global_number, be32_to_cpu(common->version));
136

137
	if (data->brdgCtl)
138
		pr_info("brdgCtl:     %08x\n",
139
			be32_to_cpu(data->brdgCtl));
140
141
	if (data->portStatusReg || data->rootCmplxStatus ||
	    data->busAgentStatus)
142
		pr_info("UtlSts:      %08x %08x %08x\n",
143
144
145
			be32_to_cpu(data->portStatusReg),
			be32_to_cpu(data->rootCmplxStatus),
			be32_to_cpu(data->busAgentStatus));
146
147
148
	if (data->deviceStatus || data->slotStatus   ||
	    data->linkStatus   || data->devCmdStatus ||
	    data->devSecStatus)
149
		pr_info("RootSts:     %08x %08x %08x %08x %08x\n",
150
151
152
153
154
			be32_to_cpu(data->deviceStatus),
			be32_to_cpu(data->slotStatus),
			be32_to_cpu(data->linkStatus),
			be32_to_cpu(data->devCmdStatus),
			be32_to_cpu(data->devSecStatus));
155
156
	if (data->rootErrorStatus   || data->uncorrErrorStatus ||
	    data->corrErrorStatus)
157
		pr_info("RootErrSts:  %08x %08x %08x\n",
158
159
160
			be32_to_cpu(data->rootErrorStatus),
			be32_to_cpu(data->uncorrErrorStatus),
			be32_to_cpu(data->corrErrorStatus));
161
162
	if (data->tlpHdr1 || data->tlpHdr2 ||
	    data->tlpHdr3 || data->tlpHdr4)
163
		pr_info("RootErrLog:  %08x %08x %08x %08x\n",
164
165
166
167
			be32_to_cpu(data->tlpHdr1),
			be32_to_cpu(data->tlpHdr2),
			be32_to_cpu(data->tlpHdr3),
			be32_to_cpu(data->tlpHdr4));
168
169
	if (data->sourceId || data->errorClass ||
	    data->correlator)
170
		pr_info("RootErrLog1: %08x %016llx %016llx\n",
171
172
173
			be32_to_cpu(data->sourceId),
			be64_to_cpu(data->errorClass),
			be64_to_cpu(data->correlator));
174
	if (data->p7iocPlssr || data->p7iocCsr)
175
		pr_info("PhbSts:      %016llx %016llx\n",
176
177
			be64_to_cpu(data->p7iocPlssr),
			be64_to_cpu(data->p7iocCsr));
178
179
	if (data->lemFir)
		pr_info("Lem:         %016llx %016llx %016llx\n",
180
181
182
			be64_to_cpu(data->lemFir),
			be64_to_cpu(data->lemErrorMask),
			be64_to_cpu(data->lemWOF));
183
184
	if (data->phbErrorStatus)
		pr_info("PhbErr:      %016llx %016llx %016llx %016llx\n",
185
186
187
188
			be64_to_cpu(data->phbErrorStatus),
			be64_to_cpu(data->phbFirstErrorStatus),
			be64_to_cpu(data->phbErrorLog0),
			be64_to_cpu(data->phbErrorLog1));
189
190
	if (data->mmioErrorStatus)
		pr_info("OutErr:      %016llx %016llx %016llx %016llx\n",
191
192
193
194
			be64_to_cpu(data->mmioErrorStatus),
			be64_to_cpu(data->mmioFirstErrorStatus),
			be64_to_cpu(data->mmioErrorLog0),
			be64_to_cpu(data->mmioErrorLog1));
195
196
	if (data->dma0ErrorStatus)
		pr_info("InAErr:      %016llx %016llx %016llx %016llx\n",
197
198
199
200
			be64_to_cpu(data->dma0ErrorStatus),
			be64_to_cpu(data->dma0FirstErrorStatus),
			be64_to_cpu(data->dma0ErrorLog0),
			be64_to_cpu(data->dma0ErrorLog1));
201
202
	if (data->dma1ErrorStatus)
		pr_info("InBErr:      %016llx %016llx %016llx %016llx\n",
203
204
205
206
			be64_to_cpu(data->dma1ErrorStatus),
			be64_to_cpu(data->dma1FirstErrorStatus),
			be64_to_cpu(data->dma1ErrorLog0),
			be64_to_cpu(data->dma1ErrorLog1));
207
208
209
210
211

	for (i = 0; i < OPAL_P7IOC_NUM_PEST_REGS; i++) {
		if ((data->pestA[i] >> 63) == 0 &&
		    (data->pestB[i] >> 63) == 0)
			continue;
212

213
		pr_info("PE[%3d] A/B: %016llx %016llx\n",
214
215
			i, be64_to_cpu(data->pestA[i]),
			be64_to_cpu(data->pestB[i]));
216
217
218
	}
}

219
220
static void pnv_pci_dump_phb3_diag_data(struct pci_controller *hose,
					struct OpalIoPhbErrorCommon *common)
221
{
222
223
224
225
	struct OpalIoPhb3ErrorData *data;
	int i;

	data = (struct OpalIoPhb3ErrorData*)common;
226
	pr_info("PHB3 PHB#%d Diag-data (Version: %d)\n",
227
		hose->global_number, be32_to_cpu(common->version));
228
	if (data->brdgCtl)
229
		pr_info("brdgCtl:     %08x\n",
230
			be32_to_cpu(data->brdgCtl));
231
232
	if (data->portStatusReg || data->rootCmplxStatus ||
	    data->busAgentStatus)
233
		pr_info("UtlSts:      %08x %08x %08x\n",
234
235
236
			be32_to_cpu(data->portStatusReg),
			be32_to_cpu(data->rootCmplxStatus),
			be32_to_cpu(data->busAgentStatus));
237
238
239
	if (data->deviceStatus || data->slotStatus   ||
	    data->linkStatus   || data->devCmdStatus ||
	    data->devSecStatus)
240
		pr_info("RootSts:     %08x %08x %08x %08x %08x\n",
241
242
243
244
245
			be32_to_cpu(data->deviceStatus),
			be32_to_cpu(data->slotStatus),
			be32_to_cpu(data->linkStatus),
			be32_to_cpu(data->devCmdStatus),
			be32_to_cpu(data->devSecStatus));
246
247
	if (data->rootErrorStatus || data->uncorrErrorStatus ||
	    data->corrErrorStatus)
248
		pr_info("RootErrSts:  %08x %08x %08x\n",
249
250
251
			be32_to_cpu(data->rootErrorStatus),
			be32_to_cpu(data->uncorrErrorStatus),
			be32_to_cpu(data->corrErrorStatus));
252
253
	if (data->tlpHdr1 || data->tlpHdr2 ||
	    data->tlpHdr3 || data->tlpHdr4)
254
		pr_info("RootErrLog:  %08x %08x %08x %08x\n",
255
256
257
258
			be32_to_cpu(data->tlpHdr1),
			be32_to_cpu(data->tlpHdr2),
			be32_to_cpu(data->tlpHdr3),
			be32_to_cpu(data->tlpHdr4));
259
260
	if (data->sourceId || data->errorClass ||
	    data->correlator)
261
		pr_info("RootErrLog1: %08x %016llx %016llx\n",
262
263
264
			be32_to_cpu(data->sourceId),
			be64_to_cpu(data->errorClass),
			be64_to_cpu(data->correlator));
265
266
	if (data->nFir)
		pr_info("nFir:        %016llx %016llx %016llx\n",
267
268
269
			be64_to_cpu(data->nFir),
			be64_to_cpu(data->nFirMask),
			be64_to_cpu(data->nFirWOF));
270
	if (data->phbPlssr || data->phbCsr)
271
		pr_info("PhbSts:      %016llx %016llx\n",
272
273
			be64_to_cpu(data->phbPlssr),
			be64_to_cpu(data->phbCsr));
274
275
	if (data->lemFir)
		pr_info("Lem:         %016llx %016llx %016llx\n",
276
277
278
			be64_to_cpu(data->lemFir),
			be64_to_cpu(data->lemErrorMask),
			be64_to_cpu(data->lemWOF));
279
280
	if (data->phbErrorStatus)
		pr_info("PhbErr:      %016llx %016llx %016llx %016llx\n",
281
282
283
284
			be64_to_cpu(data->phbErrorStatus),
			be64_to_cpu(data->phbFirstErrorStatus),
			be64_to_cpu(data->phbErrorLog0),
			be64_to_cpu(data->phbErrorLog1));
285
286
	if (data->mmioErrorStatus)
		pr_info("OutErr:      %016llx %016llx %016llx %016llx\n",
287
288
289
290
			be64_to_cpu(data->mmioErrorStatus),
			be64_to_cpu(data->mmioFirstErrorStatus),
			be64_to_cpu(data->mmioErrorLog0),
			be64_to_cpu(data->mmioErrorLog1));
291
292
	if (data->dma0ErrorStatus)
		pr_info("InAErr:      %016llx %016llx %016llx %016llx\n",
293
294
295
296
			be64_to_cpu(data->dma0ErrorStatus),
			be64_to_cpu(data->dma0FirstErrorStatus),
			be64_to_cpu(data->dma0ErrorLog0),
			be64_to_cpu(data->dma0ErrorLog1));
297
298
	if (data->dma1ErrorStatus)
		pr_info("InBErr:      %016llx %016llx %016llx %016llx\n",
299
300
301
302
			be64_to_cpu(data->dma1ErrorStatus),
			be64_to_cpu(data->dma1FirstErrorStatus),
			be64_to_cpu(data->dma1ErrorLog0),
			be64_to_cpu(data->dma1ErrorLog1));
303
304

	for (i = 0; i < OPAL_PHB3_NUM_PEST_REGS; i++) {
305
306
		if ((be64_to_cpu(data->pestA[i]) >> 63) == 0 &&
		    (be64_to_cpu(data->pestB[i]) >> 63) == 0)
307
308
			continue;

309
		pr_info("PE[%3d] A/B: %016llx %016llx\n",
310
311
				i, be64_to_cpu(data->pestA[i]),
				be64_to_cpu(data->pestB[i]));
312
313
314
315
316
317
318
319
320
321
322
323
	}
}

void pnv_pci_dump_phb_diag_data(struct pci_controller *hose,
				unsigned char *log_buff)
{
	struct OpalIoPhbErrorCommon *common;

	if (!hose || !log_buff)
		return;

	common = (struct OpalIoPhbErrorCommon *)log_buff;
324
	switch (be32_to_cpu(common->ioType)) {
325
326
327
328
329
	case OPAL_PHB_ERROR_DATA_TYPE_P7IOC:
		pnv_pci_dump_p7ioc_diag_data(hose, common);
		break;
	case OPAL_PHB_ERROR_DATA_TYPE_PHB3:
		pnv_pci_dump_phb3_diag_data(hose, common);
330
331
		break;
	default:
332
		pr_warn("%s: Unrecognized ioType %d\n",
333
			__func__, be32_to_cpu(common->ioType));
334
335
336
337
338
339
	}
}

static void pnv_pci_handle_eeh_config(struct pnv_phb *phb, u32 pe_no)
{
	unsigned long flags, rc;
340
	int has_diag, ret = 0;
341
342
343

	spin_lock_irqsave(&phb->lock, flags);

344
	/* Fetch PHB diag-data */
345
346
	rc = opal_pci_get_phb_diag_data2(phb->opal_id, phb->diag.blob,
					 PNV_PCI_DIAG_BUF_SIZE);
347
348
	has_diag = (rc == OPAL_SUCCESS);

349
350
351
352
	/* If PHB supports compound PE, to handle it */
	if (phb->unfreeze_pe) {
		ret = phb->unfreeze_pe(phb,
				       pe_no,
353
				       OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
354
355
356
357
358
359
360
361
362
363
364
	} else {
		rc = opal_pci_eeh_freeze_clear(phb->opal_id,
					     pe_no,
					     OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
		if (rc) {
			pr_warn("%s: Failure %ld clearing frozen "
				"PHB#%x-PE#%x\n",
				__func__, rc, phb->hose->global_number,
				pe_no);
			ret = -EIO;
		}
365
366
	}

367
368
369
370
371
372
373
374
375
	/*
	 * For now, let's only display the diag buffer when we fail to clear
	 * the EEH status. We'll do more sensible things later when we have
	 * proper EEH support. We need to make sure we don't pollute ourselves
	 * with the normal errors generated when probing empty slots
	 */
	if (has_diag && ret)
		pnv_pci_dump_phb_diag_data(phb->hose, phb->diag.blob);

376
377
378
	spin_unlock_irqrestore(&phb->lock, flags);
}

379
380
static void pnv_pci_config_check_eeh(struct pnv_phb *phb,
				     struct device_node *dn)
381
382
{
	u8	fstate;
383
	__be16	pcierr;
384
385
	int	pe_no;
	s64	rc;
386

387
388
389
	/*
	 * Get the PE#. During the PCI probe stage, we might not
	 * setup that yet. So all ER errors should be mapped to
390
	 * reserved PE.
391
392
	 */
	pe_no = PCI_DN(dn)->pe_number;
393
394
395
396
397
398
	if (pe_no == IODA_INVALID_PE) {
		if (phb->type == PNV_PHB_P5IOC2)
			pe_no = 0;
		else
			pe_no = phb->ioda.reserved_pe;
	}
399

400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
	/*
	 * Fetch frozen state. If the PHB support compound PE,
	 * we need handle that case.
	 */
	if (phb->get_pe_state) {
		fstate = phb->get_pe_state(phb, pe_no);
	} else {
		rc = opal_pci_eeh_freeze_status(phb->opal_id,
						pe_no,
						&fstate,
						&pcierr,
						NULL);
		if (rc) {
			pr_warn("%s: Failure %lld getting PHB#%x-PE#%x state\n",
				__func__, rc, phb->hose->global_number, pe_no);
			return;
		}
417
	}
418

419
420
421
	cfg_dbg(" -> EEH check, bdfn=%04x PE#%d fstate=%x\n",
		(PCI_DN(dn)->busno << 8) | (PCI_DN(dn)->devfn),
		pe_no, fstate);
422
423
424
425
426
427
428
429
430
431
432
433

	/* Clear the frozen state if applicable */
	if (fstate == OPAL_EEH_STOPPED_MMIO_FREEZE ||
	    fstate == OPAL_EEH_STOPPED_DMA_FREEZE  ||
	    fstate == OPAL_EEH_STOPPED_MMIO_DMA_FREEZE) {
		/*
		 * If PHB supports compound PE, freeze it for
		 * consistency.
		 */
		if (phb->freeze_pe)
			phb->freeze_pe(phb, pe_no);

434
		pnv_pci_handle_eeh_config(phb, pe_no);
435
	}
436
437
}

438
439
int pnv_pci_cfg_read(struct device_node *dn,
		     int where, int size, u32 *val)
440
{
441
442
443
	struct pci_dn *pdn = PCI_DN(dn);
	struct pnv_phb *phb = pdn->phb->private_data;
	u32 bdfn = (pdn->busno << 8) | pdn->devfn;
444
445
446
447
448
449
450
451
452
453
	s64 rc;

	switch (size) {
	case 1: {
		u8 v8;
		rc = opal_pci_config_read_byte(phb->opal_id, bdfn, where, &v8);
		*val = (rc == OPAL_SUCCESS) ? v8 : 0xff;
		break;
	}
	case 2: {
454
		__be16 v16;
455
456
		rc = opal_pci_config_read_half_word(phb->opal_id, bdfn, where,
						   &v16);
457
		*val = (rc == OPAL_SUCCESS) ? be16_to_cpu(v16) : 0xffff;
458
459
460
		break;
	}
	case 4: {
461
		__be32 v32;
462
		rc = opal_pci_config_read_word(phb->opal_id, bdfn, where, &v32);
463
		*val = (rc == OPAL_SUCCESS) ? be32_to_cpu(v32) : 0xffffffff;
464
465
466
467
468
		break;
	}
	default:
		return PCIBIOS_FUNC_NOT_SUPPORTED;
	}
469

470
471
	cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
		__func__, pdn->busno, pdn->devfn, where, size, *val);
472
473
474
	return PCIBIOS_SUCCESSFUL;
}

475
476
int pnv_pci_cfg_write(struct device_node *dn,
		      int where, int size, u32 val)
477
{
478
479
480
	struct pci_dn *pdn = PCI_DN(dn);
	struct pnv_phb *phb = pdn->phb->private_data;
	u32 bdfn = (pdn->busno << 8) | pdn->devfn;
481

482
483
	cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
		pdn->busno, pdn->devfn, where, size, val);
484
485
486
487
488
489
490
491
492
493
494
495
496
	switch (size) {
	case 1:
		opal_pci_config_write_byte(phb->opal_id, bdfn, where, val);
		break;
	case 2:
		opal_pci_config_write_half_word(phb->opal_id, bdfn, where, val);
		break;
	case 4:
		opal_pci_config_write_word(phb->opal_id, bdfn, where, val);
		break;
	default:
		return PCIBIOS_FUNC_NOT_SUPPORTED;
	}
497

498
499
500
501
502
503
504
505
506
507
508
	return PCIBIOS_SUCCESSFUL;
}

#if CONFIG_EEH
static bool pnv_pci_cfg_check(struct pci_controller *hose,
			      struct device_node *dn)
{
	struct eeh_dev *edev = NULL;
	struct pnv_phb *phb = hose->private_data;

	/* EEH not enabled ? */
509
	if (!(phb->flags & PNV_PHB_FLAG_EEH))
510
		return true;
511

512
	/* PE reset or device removed ? */
513
	edev = of_node_to_eeh_dev(dn);
514
515
516
517
518
519
520
521
	if (edev) {
		if (edev->pe &&
		    (edev->pe->state & EEH_PE_RESET))
			return false;

		if (edev->mode & EEH_DEV_REMOVED)
			return false;
	}
522
523
524
525
526
527
528
529

	return true;
}
#else
static inline pnv_pci_cfg_check(struct pci_controller *hose,
				struct device_node *dn)
{
	return true;
530
}
531
#endif /* CONFIG_EEH */
532

533
534
535
536
537
538
static int pnv_pci_read_config(struct pci_bus *bus,
			       unsigned int devfn,
			       int where, int size, u32 *val)
{
	struct device_node *dn, *busdn = pci_bus_to_OF_node(bus);
	struct pci_dn *pdn;
539
540
541
	struct pnv_phb *phb;
	bool found = false;
	int ret;
542

543
	*val = 0xFFFFFFFF;
544
545
	for (dn = busdn->child; dn; dn = dn->sibling) {
		pdn = PCI_DN(dn);
546
547
548
549
550
		if (pdn && pdn->devfn == devfn) {
			phb = pdn->phb->private_data;
			found = true;
			break;
		}
551
552
	}

553
554
555
556
557
558
559
560
561
562
563
	if (!found || !pnv_pci_cfg_check(pdn->phb, dn))
		return PCIBIOS_DEVICE_NOT_FOUND;

	ret = pnv_pci_cfg_read(dn, where, size, val);
	if (phb->flags & PNV_PHB_FLAG_EEH) {
		if (*val == EEH_IO_ERROR_VALUE(size) &&
		    eeh_dev_check_failure(of_node_to_eeh_dev(dn)))
                        return PCIBIOS_DEVICE_NOT_FOUND;
	} else {
		pnv_pci_config_check_eeh(phb, dn);
	}
564

565
	return ret;
566
567
568
569
570
571
572
573
}

static int pnv_pci_write_config(struct pci_bus *bus,
				unsigned int devfn,
				int where, int size, u32 val)
{
	struct device_node *dn, *busdn = pci_bus_to_OF_node(bus);
	struct pci_dn *pdn;
574
575
576
	struct pnv_phb *phb;
	bool found = false;
	int ret;
577
578
579

	for (dn = busdn->child; dn; dn = dn->sibling) {
		pdn = PCI_DN(dn);
580
581
582
583
584
		if (pdn && pdn->devfn == devfn) {
			phb = pdn->phb->private_data;
			found = true;
			break;
		}
585
586
	}

587
588
589
590
591
592
593
594
	if (!found || !pnv_pci_cfg_check(pdn->phb, dn))
		return PCIBIOS_DEVICE_NOT_FOUND;

	ret = pnv_pci_cfg_write(dn, where, size, val);
	if (!(phb->flags & PNV_PHB_FLAG_EEH))
		pnv_pci_config_check_eeh(phb, dn);

	return ret;
595
596
}

597
struct pci_ops pnv_pci_ops = {
598
	.read  = pnv_pci_read_config,
599
600
601
602
603
	.write = pnv_pci_write_config,
};

static int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
			 unsigned long uaddr, enum dma_data_direction direction,
604
			 struct dma_attrs *attrs, bool rm)
605
606
{
	u64 proto_tce;
607
	__be64 *tcep, *tces;
608
609
610
611
612
613
614
	u64 rpn;

	proto_tce = TCE_PCI_READ; // Read allowed

	if (direction != DMA_TO_DEVICE)
		proto_tce |= TCE_PCI_WRITE;

615
	tces = tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset;
616
	rpn = __pa(uaddr) >> tbl->it_page_shift;
617

618
	while (npages--)
619
620
		*(tcep++) = cpu_to_be64(proto_tce |
				(rpn++ << tbl->it_page_shift));
621
622
623
624
625
626

	/* Some implementations won't cache invalid TCEs and thus may not
	 * need that flush. We'll probably turn it_type into a bit mask
	 * of flags if that becomes the case
	 */
	if (tbl->it_type & TCE_PCI_SWINV_CREATE)
627
		pnv_pci_ioda_tce_invalidate(tbl, tces, tcep - 1, rm);
628
629
630
631

	return 0;
}

632
633
634
635
636
637
638
639
640
641
642
static int pnv_tce_build_vm(struct iommu_table *tbl, long index, long npages,
			    unsigned long uaddr,
			    enum dma_data_direction direction,
			    struct dma_attrs *attrs)
{
	return pnv_tce_build(tbl, index, npages, uaddr, direction, attrs,
			false);
}

static void pnv_tce_free(struct iommu_table *tbl, long index, long npages,
		bool rm)
643
{
644
	__be64 *tcep, *tces;
645

646
	tces = tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset;
647
648

	while (npages--)
649
		*(tcep++) = cpu_to_be64(0);
650

651
	if (tbl->it_type & TCE_PCI_SWINV_FREE)
652
653
654
655
656
657
		pnv_pci_ioda_tce_invalidate(tbl, tces, tcep - 1, rm);
}

static void pnv_tce_free_vm(struct iommu_table *tbl, long index, long npages)
{
	pnv_tce_free(tbl, index, npages, false);
658
659
}

660
661
662
663
664
static unsigned long pnv_tce_get(struct iommu_table *tbl, long index)
{
	return ((u64 *)tbl->it_base)[index - tbl->it_offset];
}

665
666
667
668
669
670
671
672
673
674
675
676
677
static int pnv_tce_build_rm(struct iommu_table *tbl, long index, long npages,
			    unsigned long uaddr,
			    enum dma_data_direction direction,
			    struct dma_attrs *attrs)
{
	return pnv_tce_build(tbl, index, npages, uaddr, direction, attrs, true);
}

static void pnv_tce_free_rm(struct iommu_table *tbl, long index, long npages)
{
	pnv_tce_free(tbl, index, npages, true);
}

678
679
void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
			       void *tce_mem, u64 tce_size,
680
			       u64 dma_offset, unsigned page_shift)
681
682
683
{
	tbl->it_blocksize = 16;
	tbl->it_base = (unsigned long)tce_mem;
684
	tbl->it_page_shift = page_shift;
685
	tbl->it_offset = dma_offset >> tbl->it_page_shift;
686
687
688
689
690
691
	tbl->it_index = 0;
	tbl->it_size = tce_size >> 3;
	tbl->it_busno = 0;
	tbl->it_type = TCE_PCI;
}

692
static struct iommu_table *pnv_pci_setup_bml_iommu(struct pci_controller *hose)
693
694
{
	struct iommu_table *tbl;
695
696
	const __be64 *basep, *swinvp;
	const __be32 *sizep;
697
698
699
700

	basep = of_get_property(hose->dn, "linux,tce-base", NULL);
	sizep = of_get_property(hose->dn, "linux,tce-size", NULL);
	if (basep == NULL || sizep == NULL) {
701
702
		pr_err("PCI: %s has missing tce entries !\n",
		       hose->dn->full_name);
703
704
705
706
707
708
		return NULL;
	}
	tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, hose->node);
	if (WARN_ON(!tbl))
		return NULL;
	pnv_pci_setup_iommu_table(tbl, __va(be64_to_cpup(basep)),
709
				  be32_to_cpup(sizep), 0, IOMMU_PAGE_SHIFT_4K);
710
	iommu_init_table(tbl, hose->node);
711
	iommu_register_group(tbl, pci_domain_nr(hose->bus), 0);
712
713
714
715
716

	/* Deal with SW invalidated TCEs when needed (BML way) */
	swinvp = of_get_property(hose->dn, "linux,tce-sw-invalidate-info",
				 NULL);
	if (swinvp) {
717
		tbl->it_busno = be64_to_cpu(swinvp[1]);
718
		tbl->it_index = (unsigned long)ioremap(be64_to_cpup(swinvp), 8);
719
720
		tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE;
	}
721
722
723
	return tbl;
}

724
725
static void pnv_pci_dma_fallback_setup(struct pci_controller *hose,
				       struct pci_dev *pdev)
726
727
728
729
730
731
732
733
734
735
736
{
	struct device_node *np = pci_bus_to_OF_node(hose->bus);
	struct pci_dn *pdn;

	if (np == NULL)
		return;
	pdn = PCI_DN(np);
	if (!pdn->iommu_table)
		pdn->iommu_table = pnv_pci_setup_bml_iommu(hose);
	if (!pdn->iommu_table)
		return;
737
	set_iommu_table_base_and_group(&pdev->dev, pdn->iommu_table);
738
739
}

740
static void pnv_pci_dma_dev_setup(struct pci_dev *pdev)
741
742
743
744
745
746
747
748
749
750
751
752
753
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;

	/* If we have no phb structure, try to setup a fallback based on
	 * the device-tree (RTAS PCI for example)
	 */
	if (phb && phb->dma_dev_setup)
		phb->dma_dev_setup(phb, pdev);
	else
		pnv_pci_dma_fallback_setup(hose, pdev);
}

754
755
756
757
758
759
760
761
762
763
int pnv_pci_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
{
	struct pci_controller *hose = pci_bus_to_host(pdev->bus);
	struct pnv_phb *phb = hose->private_data;

	if (phb && phb->dma_set_mask)
		return phb->dma_set_mask(phb, pdev, dma_mask);
	return __dma_set_mask(&pdev->dev, dma_mask);
}

764
765
766
767
768
769
770
771
772
773
774
775
void pnv_pci_shutdown(void)
{
	struct pci_controller *hose;

	list_for_each_entry(hose, &hose_list, list_node) {
		struct pnv_phb *phb = hose->private_data;

		if (phb && phb->shutdown)
			phb->shutdown(phb);
	}
}

Gavin Shan's avatar
Gavin Shan committed
776
/* Fixup wrong class code in p7ioc and p8 root complex */
777
static void pnv_p7ioc_rc_quirk(struct pci_dev *dev)
778
779
780
781
782
{
	dev->class = PCI_CLASS_BRIDGE_PCI << 8;
}
DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_IBM, 0x3b9, pnv_p7ioc_rc_quirk);

783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
static int pnv_pci_probe_mode(struct pci_bus *bus)
{
	struct pci_controller *hose = pci_bus_to_host(bus);
	const __be64 *tstamp;
	u64 now, target;


	/* We hijack this as a way to ensure we have waited long
	 * enough since the reset was lifted on the PCI bus
	 */
	if (bus != hose->bus)
		return PCI_PROBE_NORMAL;
	tstamp = of_get_property(hose->dn, "reset-clear-timestamp", NULL);
	if (!tstamp || !*tstamp)
		return PCI_PROBE_NORMAL;

	now = mftb() / tb_ticks_per_usec;
	target = (be64_to_cpup(tstamp) / tb_ticks_per_usec)
		+ PCI_RESET_DELAY_US;

	pr_devel("pci %04d: Reset target: 0x%llx now: 0x%llx\n",
		 hose->global_number, target, now);

	if (now < target)
		msleep((target - now + 999) / 1000);

	return PCI_PROBE_NORMAL;
}

812
813
814
815
void __init pnv_pci_init(void)
{
	struct device_node *np;

816
	pci_add_flags(PCI_CAN_SKIP_ISA_ALIGN);
817
818
819
820
821
822
823

	/* OPAL absent, try POPAL first then RTAS detection of PHBs */
	if (!firmware_has_feature(FW_FEATURE_OPAL)) {
#ifdef CONFIG_PPC_POWERNV_RTAS
		init_pci_config_tokens();
		find_and_init_phbs();
#endif /* CONFIG_PPC_POWERNV_RTAS */
824
825
826
827
828
829
830
831
832
833
834
835
836
	}
	/* OPAL is here, do our normal stuff */
	else {
		int found_ioda = 0;

		/* Look for IODA IO-Hubs. We don't support mixing IODA
		 * and p5ioc2 due to the need to change some global
		 * probing flags
		 */
		for_each_compatible_node(np, NULL, "ibm,ioda-hub") {
			pnv_pci_init_ioda_hub(np);
			found_ioda = 1;
		}
837
838

		/* Look for p5ioc2 IO-Hubs */
839
840
841
		if (!found_ioda)
			for_each_compatible_node(np, NULL, "ibm,p5ioc2")
				pnv_pci_init_p5ioc2_hub(np);
Gavin Shan's avatar
Gavin Shan committed
842
843
844
845

		/* Look for ioda2 built-in PHB3's */
		for_each_compatible_node(np, NULL, "ibm,ioda2-phb")
			pnv_pci_init_ioda2_phb(np);
846
847
848
849
850
851
852
	}

	/* Setup the linkage between OF nodes and PHBs */
	pci_devs_phb_init();

	/* Configure IOMMU DMA hooks */
	ppc_md.pci_dma_dev_setup = pnv_pci_dma_dev_setup;
853
854
855
856
	ppc_md.tce_build = pnv_tce_build_vm;
	ppc_md.tce_free = pnv_tce_free_vm;
	ppc_md.tce_build_rm = pnv_tce_build_rm;
	ppc_md.tce_free_rm = pnv_tce_free_rm;
857
	ppc_md.tce_get = pnv_tce_get;
858
	ppc_md.pci_probe_mode = pnv_pci_probe_mode;
859
860
	set_pci_dma_ops(&dma_iommu_ops);

861
862
863
864
865
866
	/* Configure MSIs */
#ifdef CONFIG_PCI_MSI
	ppc_md.msi_check_device = pnv_msi_check_device;
	ppc_md.setup_msi_irqs = pnv_setup_msi_irqs;
	ppc_md.teardown_msi_irqs = pnv_teardown_msi_irqs;
#endif
867
}
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894

static int tce_iommu_bus_notifier(struct notifier_block *nb,
		unsigned long action, void *data)
{
	struct device *dev = data;

	switch (action) {
	case BUS_NOTIFY_ADD_DEVICE:
		return iommu_add_device(dev);
	case BUS_NOTIFY_DEL_DEVICE:
		if (dev->iommu_group)
			iommu_del_device(dev);
		return 0;
	default:
		return 0;
	}
}

static struct notifier_block tce_iommu_bus_nb = {
	.notifier_call = tce_iommu_bus_notifier,
};

static int __init tce_iommu_bus_notifier_init(void)
{
	bus_register_notifier(&pci_bus_type, &tce_iommu_bus_nb);
	return 0;
}
895
machine_subsys_initcall_sync(powernv, tce_iommu_bus_notifier_init);