Commit 687d6809 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge git://git.infradead.org/~dwmw2/iommu-2.6.31

* git://git.infradead.org/~dwmw2/iommu-2.6.31:
  intel-iommu: Fix one last ia64 build problem in Pass Through Support
  VT-d: support the device IOTLB
  VT-d: cleanup iommu_flush_iotlb_psi and flush_unmaps
  VT-d: add device IOTLB invalidation support
  VT-d: parse ATSR in DMA Remapping Reporting Structure
  PCI: handle Virtual Function ATS enabling
  PCI: support the ATS capability
  intel-iommu: dmar_set_interrupt return error value
  intel-iommu: Tidy up iommu->gcmd handling
  intel-iommu: Fix tiny theoretical race in write-buffer flush.
  intel-iommu: Clean up handling of "caching mode" vs. IOTLB flushing.
  intel-iommu: Clean up handling of "caching mode" vs. context flushing.
  VT-d: fix invalid domain id for KVM context flush
  Fix !CONFIG_DMAR build failure introduced by Intel IOMMU Pass Through Support
  Intel IOMMU Pass Through Support

Fix up trivial conflicts in drivers/pci/{intel-iommu.c,intr_remapping.c}
parents 10534140 008fe148
......@@ -1006,6 +1006,7 @@ and is between 256 and 4096 characters. It is defined in the file
nomerge
forcesac
soft
pt [x86, IA64]
io7= [HW] IO7 for Marvel based alpha systems
See comment before marvel_specify_io7 in
......
......@@ -9,6 +9,11 @@ extern void pci_iommu_shutdown(void);
extern void no_iommu_init(void);
extern int force_iommu, no_iommu;
extern int iommu_detected;
#ifdef CONFIG_DMAR
extern int iommu_pass_through;
#else
#define iommu_pass_through (0)
#endif
extern void iommu_dma_init(void);
extern void machvec_init(const char *name);
......
......@@ -32,6 +32,8 @@ int force_iommu __read_mostly = 1;
int force_iommu __read_mostly;
#endif
int iommu_pass_through;
/* Dummy device used for NULL arguments (normally ISA). Better would
be probably a smaller DMA mask, but this is bug-to-bug compatible
to i386. */
......
......@@ -46,7 +46,7 @@ void __init swiotlb_dma_init(void)
void __init pci_swiotlb_init(void)
{
if (!iommu_detected) {
if (!iommu_detected || iommu_pass_through) {
#ifdef CONFIG_IA64_GENERIC
swiotlb = 1;
printk(KERN_INFO "PCI-DMA: Re-initialize machine vector.\n");
......
......@@ -6,6 +6,7 @@ extern void no_iommu_init(void);
extern struct dma_map_ops nommu_dma_ops;
extern int force_iommu, no_iommu;
extern int iommu_detected;
extern int iommu_pass_through;
/* 10 seconds */
#define DMAR_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000)
......
......@@ -32,6 +32,8 @@ int no_iommu __read_mostly;
/* Set this to 1 if there is a HW IOMMU in the system */
int iommu_detected __read_mostly = 0;
int iommu_pass_through;
dma_addr_t bad_dma_address __read_mostly = 0;
EXPORT_SYMBOL(bad_dma_address);
......@@ -209,6 +211,10 @@ static __init int iommu_setup(char *p)
#ifdef CONFIG_SWIOTLB
if (!strncmp(p, "soft", 4))
swiotlb = 1;
if (!strncmp(p, "pt", 2)) {
iommu_pass_through = 1;
return 1;
}
#endif
gart_parse_options(p);
......
......@@ -71,7 +71,8 @@ void __init pci_swiotlb_init(void)
{
/* don't initialize swiotlb if iommu=off (no_iommu=1) */
#ifdef CONFIG_X86_64
if (!iommu_detected && !no_iommu && max_pfn > MAX_DMA32_PFN)
if ((!iommu_detected && !no_iommu && max_pfn > MAX_DMA32_PFN) ||
iommu_pass_through)
swiotlb = 1;
#endif
if (swiotlb_force)
......
......@@ -267,6 +267,84 @@ rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
}
return ret;
}
static LIST_HEAD(dmar_atsr_units);
static int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
{
struct acpi_dmar_atsr *atsr;
struct dmar_atsr_unit *atsru;
atsr = container_of(hdr, struct acpi_dmar_atsr, header);
atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
if (!atsru)
return -ENOMEM;
atsru->hdr = hdr;
atsru->include_all = atsr->flags & 0x1;
list_add(&atsru->list, &dmar_atsr_units);
return 0;
}
static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
{
int rc;
struct acpi_dmar_atsr *atsr;
if (atsru->include_all)
return 0;
atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
rc = dmar_parse_dev_scope((void *)(atsr + 1),
(void *)atsr + atsr->header.length,
&atsru->devices_cnt, &atsru->devices,
atsr->segment);
if (rc || !atsru->devices_cnt) {
list_del(&atsru->list);
kfree(atsru);
}
return rc;
}
int dmar_find_matched_atsr_unit(struct pci_dev *dev)
{
int i;
struct pci_bus *bus;
struct acpi_dmar_atsr *atsr;
struct dmar_atsr_unit *atsru;
list_for_each_entry(atsru, &dmar_atsr_units, list) {
atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
if (atsr->segment == pci_domain_nr(dev->bus))
goto found;
}
return 0;
found:
for (bus = dev->bus; bus; bus = bus->parent) {
struct pci_dev *bridge = bus->self;
if (!bridge || !bridge->is_pcie ||
bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
return 0;
if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) {
for (i = 0; i < atsru->devices_cnt; i++)
if (atsru->devices[i] == bridge)
return 1;
break;
}
}
if (atsru->include_all)
return 1;
return 0;
}
#endif
static void __init
......@@ -274,22 +352,28 @@ dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
{
struct acpi_dmar_hardware_unit *drhd;
struct acpi_dmar_reserved_memory *rmrr;
struct acpi_dmar_atsr *atsr;
switch (header->type) {
case ACPI_DMAR_TYPE_HARDWARE_UNIT:
drhd = (struct acpi_dmar_hardware_unit *)header;
drhd = container_of(header, struct acpi_dmar_hardware_unit,
header);
printk (KERN_INFO PREFIX
"DRHD (flags: 0x%08x)base: 0x%016Lx\n",
drhd->flags, (unsigned long long)drhd->address);
"DRHD base: %#016Lx flags: %#x\n",
(unsigned long long)drhd->address, drhd->flags);
break;
case ACPI_DMAR_TYPE_RESERVED_MEMORY:
rmrr = (struct acpi_dmar_reserved_memory *)header;
rmrr = container_of(header, struct acpi_dmar_reserved_memory,
header);
printk (KERN_INFO PREFIX
"RMRR base: 0x%016Lx end: 0x%016Lx\n",
"RMRR base: %#016Lx end: %#016Lx\n",
(unsigned long long)rmrr->base_address,
(unsigned long long)rmrr->end_address);
break;
case ACPI_DMAR_TYPE_ATSR:
atsr = container_of(header, struct acpi_dmar_atsr, header);
printk(KERN_INFO PREFIX "ATSR flags: %#x\n", atsr->flags);
break;
}
}
......@@ -361,6 +445,11 @@ parse_dmar_table(void)
case ACPI_DMAR_TYPE_RESERVED_MEMORY:
#ifdef CONFIG_DMAR
ret = dmar_parse_one_rmrr(entry_header);
#endif
break;
case ACPI_DMAR_TYPE_ATSR:
#ifdef CONFIG_DMAR
ret = dmar_parse_one_atsr(entry_header);
#endif
break;
default:
......@@ -431,11 +520,19 @@ int __init dmar_dev_scope_init(void)
#ifdef CONFIG_DMAR
{
struct dmar_rmrr_unit *rmrr, *rmrr_n;
struct dmar_atsr_unit *atsr, *atsr_n;
list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
ret = rmrr_parse_dev(rmrr);
if (ret)
return ret;
}
list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
ret = atsr_parse_dev(atsr);
if (ret)
return ret;
}
}
#endif
......@@ -468,6 +565,9 @@ int __init dmar_table_init(void)
#ifdef CONFIG_DMAR
if (list_empty(&dmar_rmrr_units))
printk(KERN_INFO PREFIX "No RMRR found\n");
if (list_empty(&dmar_atsr_units))
printk(KERN_INFO PREFIX "No ATSR found\n");
#endif
#ifdef CONFIG_INTR_REMAP
......@@ -515,6 +615,7 @@ int alloc_iommu(struct dmar_drhd_unit *drhd)
u32 ver;
static int iommu_allocated = 0;
int agaw = 0;
int msagaw = 0;
iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
if (!iommu)
......@@ -535,12 +636,20 @@ int alloc_iommu(struct dmar_drhd_unit *drhd)
agaw = iommu_calculate_agaw(iommu);
if (agaw < 0) {
printk(KERN_ERR
"Cannot get a valid agaw for iommu (seq_id = %d)\n",
"Cannot get a valid agaw for iommu (seq_id = %d)\n",
iommu->seq_id);
goto error;
}
msagaw = iommu_calculate_max_sagaw(iommu);
if (msagaw < 0) {
printk(KERN_ERR
"Cannot get a valid max agaw for iommu (seq_id = %d)\n",
iommu->seq_id);
goto error;
}
#endif
iommu->agaw = agaw;
iommu->msagaw = msagaw;
/* the registers might be more than one page */
map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
......@@ -590,7 +699,8 @@ void free_iommu(struct intel_iommu *iommu)
*/
static inline void reclaim_free_desc(struct q_inval *qi)
{
while (qi->desc_status[qi->free_tail] == QI_DONE) {
while (qi->desc_status[qi->free_tail] == QI_DONE ||
qi->desc_status[qi->free_tail] == QI_ABORT) {
qi->desc_status[qi->free_tail] = QI_FREE;
qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
qi->free_cnt++;
......@@ -600,10 +710,13 @@ static inline void reclaim_free_desc(struct q_inval *qi)
static int qi_check_fault(struct intel_iommu *iommu, int index)
{
u32 fault;
int head;
int head, tail;
struct q_inval *qi = iommu->qi;
int wait_index = (index + 1) % QI_LENGTH;
if (qi->desc_status[wait_index] == QI_ABORT)
return -EAGAIN;
fault = readl(iommu->reg + DMAR_FSTS_REG);
/*
......@@ -613,7 +726,11 @@ static int qi_check_fault(struct intel_iommu *iommu, int index)
*/
if (fault & DMA_FSTS_IQE) {
head = readl(iommu->reg + DMAR_IQH_REG);
if ((head >> 4) == index) {
if ((head >> DMAR_IQ_SHIFT) == index) {
printk(KERN_ERR "VT-d detected invalid descriptor: "
"low=%llx, high=%llx\n",
(unsigned long long)qi->desc[index].low,
(unsigned long long)qi->desc[index].high);
memcpy(&qi->desc[index], &qi->desc[wait_index],
sizeof(struct qi_desc));
__iommu_flush_cache(iommu, &qi->desc[index],
......@@ -623,6 +740,32 @@ static int qi_check_fault(struct intel_iommu *iommu, int index)
}
}
/*
* If ITE happens, all pending wait_desc commands are aborted.
* No new descriptors are fetched until the ITE is cleared.
*/
if (fault & DMA_FSTS_ITE) {
head = readl(iommu->reg + DMAR_IQH_REG);
head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
head |= 1;
tail = readl(iommu->reg + DMAR_IQT_REG);
tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
do {
if (qi->desc_status[head] == QI_IN_USE)
qi->desc_status[head] = QI_ABORT;
head = (head - 2 + QI_LENGTH) % QI_LENGTH;
} while (head != tail);
if (qi->desc_status[wait_index] == QI_ABORT)
return -EAGAIN;
}
if (fault & DMA_FSTS_ICE)
writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
return 0;
}
......@@ -632,7 +775,7 @@ static int qi_check_fault(struct intel_iommu *iommu, int index)
*/
int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
{
int rc = 0;
int rc;
struct q_inval *qi = iommu->qi;
struct qi_desc *hw, wait_desc;
int wait_index, index;
......@@ -643,6 +786,9 @@ int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
hw = qi->desc;
restart:
rc = 0;
spin_lock_irqsave(&qi->q_lock, flags);
while (qi->free_cnt < 3) {
spin_unlock_irqrestore(&qi->q_lock, flags);
......@@ -673,7 +819,7 @@ int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
* update the HW tail register indicating the presence of
* new descriptors.
*/
writel(qi->free_head << 4, iommu->reg + DMAR_IQT_REG);
writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
while (qi->desc_status[wait_index] != QI_DONE) {
/*
......@@ -685,18 +831,21 @@ int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
*/
rc = qi_check_fault(iommu, index);
if (rc)
goto out;
break;
spin_unlock(&qi->q_lock);
cpu_relax();
spin_lock(&qi->q_lock);
}
out:
qi->desc_status[index] = qi->desc_status[wait_index] = QI_DONE;
qi->desc_status[index] = QI_DONE;
reclaim_free_desc(qi);
spin_unlock_irqrestore(&qi->q_lock, flags);
if (rc == -EAGAIN)
goto restart;
return rc;
}
......@@ -714,41 +863,26 @@ void qi_global_iec(struct intel_iommu *iommu)
qi_submit_sync(&desc, iommu);
}
int qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
u64 type, int non_present_entry_flush)
void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
u64 type)
{
struct qi_desc desc;
if (non_present_entry_flush) {
if (!cap_caching_mode(iommu->cap))
return 1;
else
did = 0;
}
desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
| QI_CC_GRAN(type) | QI_CC_TYPE;
desc.high = 0;
return qi_submit_sync(&desc, iommu);
qi_submit_sync(&desc, iommu);
}
int qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
unsigned int size_order, u64 type,
int non_present_entry_flush)
void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
unsigned int size_order, u64 type)
{
u8 dw = 0, dr = 0;
struct qi_desc desc;
int ih = 0;
if (non_present_entry_flush) {
if (!cap_caching_mode(iommu->cap))
return 1;
else
did = 0;
}
if (cap_write_drain(iommu->cap))
dw = 1;
......@@ -760,7 +894,28 @@ int qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
| QI_IOTLB_AM(size_order);
return qi_submit_sync(&desc, iommu);
qi_submit_sync(&desc, iommu);
}
void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
u64 addr, unsigned mask)
{
struct qi_desc desc;
if (mask) {
BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
} else
desc.high = QI_DEV_IOTLB_ADDR(addr);
if (qdep >= QI_DEV_IOTLB_MAX_INVS)
qdep = 0;
desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
QI_DIOTLB_TYPE;
qi_submit_sync(&desc, iommu);
}
/*
......@@ -790,7 +945,6 @@ void dmar_disable_qi(struct intel_iommu *iommu)
cpu_relax();
iommu->gcmd &= ~DMA_GCMD_QIE;
writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
......@@ -804,7 +958,7 @@ end:
*/
static void __dmar_enable_qi(struct intel_iommu *iommu)
{
u32 cmd, sts;
u32 sts;
unsigned long flags;
struct q_inval *qi = iommu->qi;
......@@ -818,9 +972,8 @@ static void __dmar_enable_qi(struct intel_iommu *iommu)
dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
cmd = iommu->gcmd | DMA_GCMD_QIE;
iommu->gcmd |= DMA_GCMD_QIE;
writel(cmd, iommu->reg + DMAR_GCMD_REG);
writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
/* Make sure hardware complete it */
IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
......@@ -1096,7 +1249,7 @@ int dmar_set_interrupt(struct intel_iommu *iommu)
set_irq_data(irq, NULL);
iommu->irq = 0;
destroy_irq(irq);
return 0;
return ret;
}
ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
......
This diff is collapsed.
......@@ -409,7 +409,7 @@ int free_irte(int irq)
static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode)
{
u64 addr;
u32 cmd, sts;
u32 sts;
unsigned long flags;
addr = virt_to_phys((void *)iommu->ir_table->base);
......@@ -420,9 +420,8 @@ static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode)
(addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE);
/* Set interrupt-remapping table pointer */
cmd = iommu->gcmd | DMA_GCMD_SIRTP;
iommu->gcmd |= DMA_GCMD_SIRTP;
writel(cmd, iommu->reg + DMAR_GCMD_REG);
writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
readl, (sts & DMA_GSTS_IRTPS), sts);
......@@ -437,9 +436,8 @@ static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode)
spin_lock_irqsave(&iommu->register_lock, flags);
/* Enable interrupt-remapping */
cmd = iommu->gcmd | DMA_GCMD_IRE;
iommu->gcmd |= DMA_GCMD_IRE;
writel(cmd, iommu->reg + DMAR_GCMD_REG);
writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
readl, (sts & DMA_GSTS_IRES), sts);
......
......@@ -5,6 +5,7 @@
*
* PCI Express I/O Virtualization (IOV) support.
* Single Root IOV 1.0
* Address Translation Service 1.0
*/
#include <linux/pci.h>
......@@ -492,10 +493,10 @@ found:
if (pdev)
iov->dev = pci_dev_get(pdev);
else {
else
iov->dev = dev;
mutex_init(&iov->lock);
}
mutex_init(&iov->lock);
dev->sriov = iov;
dev->is_physfn = 1;
......@@ -515,11 +516,11 @@ static void sriov_release(struct pci_dev *dev)
{
BUG_ON(dev->sriov->nr_virtfn);
if (dev == dev->sriov->dev)
mutex_destroy(&dev->sriov->lock);
else
if (dev != dev->sriov->dev)
pci_dev_put(dev->sriov->dev);
mutex_destroy(&dev->sriov->lock);
kfree(dev->sriov);
dev->sriov = NULL;
}
......@@ -681,3 +682,145 @@ irqreturn_t pci_sriov_migration(struct pci_dev *dev)
return sriov_migration(dev) ? IRQ_HANDLED : IRQ_NONE;
}
EXPORT_SYMBOL_GPL(pci_sriov_migration);
static int ats_alloc_one(struct pci_dev *dev, int ps)
{
int pos;
u16 cap;
struct pci_ats *ats;
pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS);
if (!pos)
return -ENODEV;
ats = kzalloc(sizeof(*ats), GFP_KERNEL);
if (!ats)
return -ENOMEM;
ats->pos = pos;
ats->stu = ps;
pci_read_config_word(dev, pos + PCI_ATS_CAP, &cap);
ats->qdep = PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) :
PCI_ATS_MAX_QDEP;
dev->ats = ats;
return 0;
}
static void ats_free_one(struct pci_dev *dev)
{
kfree(dev->ats);
dev->ats = NULL;
}
/**
* pci_enable_ats - enable the ATS capability
* @dev: the PCI device
* @ps: the IOMMU page shift
*
* Returns 0 on success, or negative on failure.
*/
int pci_enable_ats(struct pci_dev *dev, int ps)
{
int rc;
u16 ctrl;
BUG_ON(dev->ats && dev->ats->is_enabled);
if (ps < PCI_ATS_MIN_STU)
return -EINVAL;
if (dev->is_physfn || dev->is_virtfn) {
struct pci_dev *pdev = dev->is_physfn ? dev : dev->physfn;
mutex_lock(&pdev->sriov->lock);
if (pdev->ats)
rc = pdev->ats->stu == ps ? 0 : -EINVAL;