Commit a7efd777 authored by Ram Amrani's avatar Ram Amrani Committed by Doug Ledford
Browse files

qedr: Add support for PD,PKEY and CQ verbs



Add support for protection domain and completion queue verbs.
Signed-off-by: default avatarRajesh Borundia <rajesh.borundia@cavium.com>
Signed-off-by: default avatarRam Amrani <Ram.Amrani@cavium.com>
Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parent ac1b36e5
......@@ -87,7 +87,14 @@ static int qedr_register_device(struct qedr_dev *dev)
dev->ibdev.uverbs_cmd_mask = QEDR_UVERBS(GET_CONTEXT) |
QEDR_UVERBS(QUERY_DEVICE) |
QEDR_UVERBS(QUERY_PORT);
QEDR_UVERBS(QUERY_PORT) |
QEDR_UVERBS(ALLOC_PD) |
QEDR_UVERBS(DEALLOC_PD) |
QEDR_UVERBS(CREATE_COMP_CHANNEL) |
QEDR_UVERBS(CREATE_CQ) |
QEDR_UVERBS(RESIZE_CQ) |
QEDR_UVERBS(DESTROY_CQ) |
QEDR_UVERBS(REQ_NOTIFY_CQ);
dev->ibdev.phys_port_cnt = 1;
dev->ibdev.num_comp_vectors = dev->num_cnq;
......@@ -105,6 +112,16 @@ static int qedr_register_device(struct qedr_dev *dev)
dev->ibdev.dealloc_ucontext = qedr_dealloc_ucontext;
dev->ibdev.mmap = qedr_mmap;
dev->ibdev.alloc_pd = qedr_alloc_pd;
dev->ibdev.dealloc_pd = qedr_dealloc_pd;
dev->ibdev.create_cq = qedr_create_cq;
dev->ibdev.destroy_cq = qedr_destroy_cq;
dev->ibdev.resize_cq = qedr_resize_cq;
dev->ibdev.req_notify_cq = qedr_arm_cq;
dev->ibdev.query_pkey = qedr_query_pkey;
dev->ibdev.dma_device = &dev->pdev->dev;
dev->ibdev.get_link_layer = qedr_link_layer;
......@@ -322,6 +339,8 @@ static irqreturn_t qedr_irq_handler(int irq, void *handle)
{
u16 hw_comp_cons, sw_comp_cons;
struct qedr_cnq *cnq = handle;
struct regpair *cq_handle;
struct qedr_cq *cq;
qed_sb_ack(cnq->sb, IGU_INT_DISABLE, 0);
......@@ -334,8 +353,36 @@ static irqreturn_t qedr_irq_handler(int irq, void *handle)
rmb();
while (sw_comp_cons != hw_comp_cons) {
cq_handle = (struct regpair *)qed_chain_consume(&cnq->pbl);
cq = (struct qedr_cq *)(uintptr_t)HILO_U64(cq_handle->hi,
cq_handle->lo);
if (cq == NULL) {
DP_ERR(cnq->dev,
"Received NULL CQ cq_handle->hi=%d cq_handle->lo=%d sw_comp_cons=%d hw_comp_cons=%d\n",
cq_handle->hi, cq_handle->lo, sw_comp_cons,
hw_comp_cons);
break;
}
if (cq->sig != QEDR_CQ_MAGIC_NUMBER) {
DP_ERR(cnq->dev,
"Problem with cq signature, cq_handle->hi=%d ch_handle->lo=%d cq=%p\n",
cq_handle->hi, cq_handle->lo, cq);
break;
}
cq->arm_flags = 0;
if (cq->ibcq.comp_handler)
(*cq->ibcq.comp_handler)
(&cq->ibcq, cq->ibcq.cq_context);
sw_comp_cons = qed_chain_get_cons_idx(&cnq->pbl);
cnq->n_comp++;
}
qed_ops->rdma_cnq_prod_update(cnq->dev->rdma_ctx, cnq->index,
......
......@@ -50,6 +50,10 @@
#define QEDR_MSG_INIT "INIT"
#define QEDR_MSG_MISC "MISC"
#define QEDR_MSG_CQ " CQ"
#define QEDR_MSG_MR " MR"
#define QEDR_CQ_MAGIC_NUMBER (0x11223344)
struct qedr_dev;
......@@ -181,6 +185,12 @@ struct qedr_dev {
#define QEDR_ROCE_PKEY_TABLE_LEN 1
#define QEDR_ROCE_PKEY_DEFAULT 0xffff
struct qedr_pbl {
struct list_head list_entry;
void *va;
dma_addr_t pa;
};
struct qedr_ucontext {
struct ib_ucontext ibucontext;
struct qedr_dev *dev;
......@@ -196,6 +206,64 @@ struct qedr_ucontext {
struct mutex mm_list_lock;
};
union db_prod64 {
struct rdma_pwm_val32_data data;
u64 raw;
};
enum qedr_cq_type {
QEDR_CQ_TYPE_GSI,
QEDR_CQ_TYPE_KERNEL,
QEDR_CQ_TYPE_USER,
};
struct qedr_pbl_info {
u32 num_pbls;
u32 num_pbes;
u32 pbl_size;
u32 pbe_size;
bool two_layered;
};
struct qedr_userq {
struct ib_umem *umem;
struct qedr_pbl_info pbl_info;
struct qedr_pbl *pbl_tbl;
u64 buf_addr;
size_t buf_len;
};
struct qedr_cq {
struct ib_cq ibcq;
enum qedr_cq_type cq_type;
u32 sig;
u16 icid;
/* Lock to protect multiplem CQ's */
spinlock_t cq_lock;
u8 arm_flags;
struct qed_chain pbl;
void __iomem *db_addr;
union db_prod64 db;
u8 pbl_toggle;
union rdma_cqe *latest_cqe;
union rdma_cqe *toggle_cqe;
u32 cq_cons;
struct qedr_userq q;
};
struct qedr_pd {
struct ib_pd ibpd;
u32 pd_id;
struct qedr_ucontext *uctx;
};
struct qedr_mm {
struct {
u64 phy_addr;
......@@ -215,4 +283,14 @@ static inline struct qedr_dev *get_qedr_dev(struct ib_device *ibdev)
return container_of(ibdev, struct qedr_dev, ibdev);
}
static inline struct qedr_pd *get_qedr_pd(struct ib_pd *ibpd)
{
return container_of(ibpd, struct qedr_pd, ibpd);
}
static inline struct qedr_cq *get_qedr_cq(struct ib_cq *ibcq)
{
return container_of(ibcq, struct qedr_cq, ibcq);
}
#endif
......@@ -47,6 +47,19 @@ struct rdma_cqe_responder {
__le32 imm_data_hi;
__le16 rq_cons;
u8 flags;
#define RDMA_CQE_RESPONDER_TOGGLE_BIT_MASK 0x1
#define RDMA_CQE_RESPONDER_TOGGLE_BIT_SHIFT 0
#define RDMA_CQE_RESPONDER_TYPE_MASK 0x3
#define RDMA_CQE_RESPONDER_TYPE_SHIFT 1
#define RDMA_CQE_RESPONDER_INV_FLG_MASK 0x1
#define RDMA_CQE_RESPONDER_INV_FLG_SHIFT 3
#define RDMA_CQE_RESPONDER_IMM_FLG_MASK 0x1
#define RDMA_CQE_RESPONDER_IMM_FLG_SHIFT 4
#define RDMA_CQE_RESPONDER_RDMA_FLG_MASK 0x1
#define RDMA_CQE_RESPONDER_RDMA_FLG_SHIFT 5
#define RDMA_CQE_RESPONDER_RESERVED2_MASK 0x3
#define RDMA_CQE_RESPONDER_RESERVED2_SHIFT 6
u8 status;
};
struct rdma_cqe_requester {
......@@ -58,6 +71,12 @@ struct rdma_cqe_requester {
__le32 reserved3;
__le16 reserved4;
u8 flags;
#define RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK 0x1
#define RDMA_CQE_REQUESTER_TOGGLE_BIT_SHIFT 0
#define RDMA_CQE_REQUESTER_TYPE_MASK 0x3
#define RDMA_CQE_REQUESTER_TYPE_SHIFT 1
#define RDMA_CQE_REQUESTER_RESERVED5_MASK 0x1F
#define RDMA_CQE_REQUESTER_RESERVED5_SHIFT 3
u8 status;
};
......@@ -66,6 +85,12 @@ struct rdma_cqe_common {
struct regpair qp_handle;
__le16 reserved1[7];
u8 flags;
#define RDMA_CQE_COMMON_TOGGLE_BIT_MASK 0x1
#define RDMA_CQE_COMMON_TOGGLE_BIT_SHIFT 0
#define RDMA_CQE_COMMON_TYPE_MASK 0x3
#define RDMA_CQE_COMMON_TYPE_SHIFT 1
#define RDMA_CQE_COMMON_RESERVED2_MASK 0x1F
#define RDMA_CQE_COMMON_RESERVED2_SHIFT 3
u8 status;
};
......@@ -76,6 +101,45 @@ union rdma_cqe {
struct rdma_cqe_common cmn;
};
/* * CQE requester status enumeration */
enum rdma_cqe_requester_status_enum {
RDMA_CQE_REQ_STS_OK,
RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR,
RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR,
RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR,
RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR,
RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR,
RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR,
RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR,
RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR,
RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR,
RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR,
RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR,
MAX_RDMA_CQE_REQUESTER_STATUS_ENUM
};
/* CQE responder status enumeration */
enum rdma_cqe_responder_status_enum {
RDMA_CQE_RESP_STS_OK,
RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR,
RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR,
RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR,
RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR,
RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR,
RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR,
RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR,
MAX_RDMA_CQE_RESPONDER_STATUS_ENUM
};
/* CQE type enumeration */
enum rdma_cqe_type {
RDMA_CQE_TYPE_REQUESTER,
RDMA_CQE_TYPE_RESPONDER_RQ,
RDMA_CQE_TYPE_RESPONDER_SRQ,
RDMA_CQE_TYPE_INVALID,
MAX_RDMA_CQE_TYPE
};
struct rdma_sq_sge {
__le32 length;
struct regpair addr;
......@@ -93,4 +157,19 @@ struct rdma_srq_sge {
__le32 length;
__le32 l_key;
};
/* Rdma doorbell data for CQ */
struct rdma_pwm_val32_data {
__le16 icid;
u8 agg_flags;
u8 params;
#define RDMA_PWM_VAL32_DATA_AGG_CMD_MASK 0x3
#define RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT 0
#define RDMA_PWM_VAL32_DATA_BYPASS_EN_MASK 0x1
#define RDMA_PWM_VAL32_DATA_BYPASS_EN_SHIFT 2
#define RDMA_PWM_VAL32_DATA_RESERVED_MASK 0x1F
#define RDMA_PWM_VAL32_DATA_RESERVED_SHIFT 3
__le32 value;
};
#endif /* __QED_HSI_RDMA__ */
......@@ -49,6 +49,17 @@
#include "verbs.h"
#include <rdma/qedr-abi.h>
#define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
{
if (index > QEDR_ROCE_PKEY_TABLE_LEN)
return -EINVAL;
*pkey = QEDR_ROCE_PKEY_DEFAULT;
return 0;
}
int qedr_query_gid(struct ib_device *ibdev, u8 port, int index,
union ib_gid *sgid)
{
......@@ -454,3 +465,531 @@ int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc);
return rc;
}
struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
struct ib_ucontext *context, struct ib_udata *udata)
{
struct qedr_dev *dev = get_qedr_dev(ibdev);
struct qedr_ucontext *uctx = NULL;
struct qedr_alloc_pd_uresp uresp;
struct qedr_pd *pd;
u16 pd_id;
int rc;
DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
(udata && context) ? "User Lib" : "Kernel");
if (!dev->rdma_ctx) {
DP_ERR(dev, "invlaid RDMA context\n");
return ERR_PTR(-EINVAL);
}
pd = kzalloc(sizeof(*pd), GFP_KERNEL);
if (!pd)
return ERR_PTR(-ENOMEM);
dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
uresp.pd_id = pd_id;
pd->pd_id = pd_id;
if (udata && context) {
rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
if (rc)
DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
uctx = get_qedr_ucontext(context);
uctx->pd = pd;
pd->uctx = uctx;
}
return &pd->ibpd;
}
int qedr_dealloc_pd(struct ib_pd *ibpd)
{
struct qedr_dev *dev = get_qedr_dev(ibpd->device);
struct qedr_pd *pd = get_qedr_pd(ibpd);
if (!pd)
pr_err("Invalid PD received in dealloc_pd\n");
DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
kfree(pd);
return 0;
}
static void qedr_free_pbl(struct qedr_dev *dev,
struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
{
struct pci_dev *pdev = dev->pdev;
int i;
for (i = 0; i < pbl_info->num_pbls; i++) {
if (!pbl[i].va)
continue;
dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
pbl[i].va, pbl[i].pa);
}
kfree(pbl);
}
#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
struct qedr_pbl_info *pbl_info,
gfp_t flags)
{
struct pci_dev *pdev = dev->pdev;
struct qedr_pbl *pbl_table;
dma_addr_t *pbl_main_tbl;
dma_addr_t pa;
void *va;
int i;
pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
if (!pbl_table)
return ERR_PTR(-ENOMEM);
for (i = 0; i < pbl_info->num_pbls; i++) {
va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size,
&pa, flags);
if (!va)
goto err;
memset(va, 0, pbl_info->pbl_size);
pbl_table[i].va = va;
pbl_table[i].pa = pa;
}
/* Two-Layer PBLs, if we have more than one pbl we need to initialize
* the first one with physical pointers to all of the rest
*/
pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
for (i = 0; i < pbl_info->num_pbls - 1; i++)
pbl_main_tbl[i] = pbl_table[i + 1].pa;
return pbl_table;
err:
for (i--; i >= 0; i--)
dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
pbl_table[i].va, pbl_table[i].pa);
qedr_free_pbl(dev, pbl_info, pbl_table);
return ERR_PTR(-ENOMEM);
}
static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
struct qedr_pbl_info *pbl_info,
u32 num_pbes, int two_layer_capable)
{
u32 pbl_capacity;
u32 pbl_size;
u32 num_pbls;
if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
if (num_pbes > MAX_PBES_TWO_LAYER) {
DP_ERR(dev, "prepare pbl table: too many pages %d\n",
num_pbes);
return -EINVAL;
}
/* calculate required pbl page size */
pbl_size = MIN_FW_PBL_PAGE_SIZE;
pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
NUM_PBES_ON_PAGE(pbl_size);
while (pbl_capacity < num_pbes) {
pbl_size *= 2;
pbl_capacity = pbl_size / sizeof(u64);
pbl_capacity = pbl_capacity * pbl_capacity;
}
num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
num_pbls++; /* One for the layer0 ( points to the pbls) */
pbl_info->two_layered = true;
} else {
/* One layered PBL */
num_pbls = 1;
pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
roundup_pow_of_two((num_pbes * sizeof(u64))));
pbl_info->two_layered = false;
}
pbl_info->num_pbls = num_pbls;
pbl_info->pbl_size = pbl_size;
pbl_info->num_pbes = num_pbes;
DP_DEBUG(dev, QEDR_MSG_MR,
"prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
return 0;
}
static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
struct qedr_pbl *pbl,
struct qedr_pbl_info *pbl_info)
{
int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
struct qedr_pbl *pbl_tbl;
struct scatterlist *sg;
struct regpair *pbe;
int entry;
u32 addr;
if (!pbl_info->num_pbes)
return;
/* If we have a two layered pbl, the first pbl points to the rest
* of the pbls and the first entry lays on the second pbl in the table
*/
if (pbl_info->two_layered)
pbl_tbl = &pbl[1];
else
pbl_tbl = pbl;
pbe = (struct regpair *)pbl_tbl->va;
if (!pbe) {
DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
return;
}
pbe_cnt = 0;
shift = ilog2(umem->page_size);
for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
pages = sg_dma_len(sg) >> shift;
for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
/* store the page address in pbe */
pbe->lo = cpu_to_le32(sg_dma_address(sg) +
umem->page_size * pg_cnt);
addr = upper_32_bits(sg_dma_address(sg) +
umem->page_size * pg_cnt);
pbe->hi = cpu_to_le32(addr);
pbe_cnt++;
total_num_pbes++;
pbe++;
if (total_num_pbes == pbl_info->num_pbes)
return;
/* If the given pbl is full storing the pbes,
* move to next pbl.
*/
if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
pbl_tbl++;
pbe = (struct regpair *)pbl_tbl->va;
pbe_cnt = 0;
}
}
}
}
static int qedr_copy_cq_uresp(struct qedr_dev *dev,
struct qedr_cq *cq, struct ib_udata *udata)
{
struct qedr_create_cq_uresp uresp;
int rc;
memset(&uresp, 0, sizeof(uresp));
uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
uresp.icid = cq->icid;
rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
if (rc)
DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
return rc;
}
static void consume_cqe(struct qedr_cq *cq)
{
if (cq->latest_cqe == cq->toggle_cqe)
cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
cq->latest_cqe = qed_chain_consume(&cq->pbl);
}
static inline int qedr_align_cq_entries(int entries)
{
u64 size, aligned_size;
/* We allocate an extra entry that we don't report to the FW. */
size = (entries + 1) * QEDR_CQE_SIZE;
aligned_size = ALIGN(size, PAGE_SIZE);
return aligned_size / QEDR_CQE_SIZE;
}
static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
struct qedr_dev *dev,
struct qedr_userq *q,
u64 buf_addr, size_t buf_len,
int access, int dmasync)
{
int page_cnt;
int rc;
q->buf_addr = buf_addr;
q->buf_len = buf_len;
q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
if (IS_ERR(q->umem)) {
DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
PTR_ERR(q->umem));
return PTR_ERR(q->umem);
}
page_cnt = ib_umem_page_count(q->umem);
rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt, 0);
if (rc)
goto err0;
q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
if (IS_ERR_OR_NULL(q->pbl_tbl))
goto err0;
qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info);
return 0;
err0:
ib_umem_release(q->umem);
return rc;
}
static inline void qedr_init_cq_params(struct qedr_cq *cq,