Commit 37cb11ac authored by Devesh Sharma's avatar Devesh Sharma Committed by Doug Ledford
Browse files

RDMA/bnxt_re: Add SRQ support for Broadcom adapters



Shared receive queue (SRQ) is defined as a pool of
receive buffers shared among multiple QPs which belong
to same protection domain in a given process context.
Use of SRQ reduces the memory foot print of IB applications.

Broadcom adapters support SRQ, adding code-changes to enable
shared receive queue.
Signed-off-by: default avatarDevesh Sharma <devesh.sharma@broadcom.com>
Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parent 89f81008
......@@ -1027,6 +1027,7 @@ struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,
struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
struct bnxt_re_qp *qp;
struct bnxt_re_cq *cq;
struct bnxt_re_srq *srq;
int rc, entries;
if ((qp_init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) ||
......@@ -1082,9 +1083,15 @@ struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,
}
if (qp_init_attr->srq) {
dev_err(rdev_to_dev(rdev), "SRQ not supported");
rc = -ENOTSUPP;
goto fail;
srq = container_of(qp_init_attr->srq, struct bnxt_re_srq,
ib_srq);
if (!srq) {
dev_err(rdev_to_dev(rdev), "SRQ not found");
rc = -EINVAL;
goto fail;
}
qp->qplib_qp.srq = &srq->qplib_srq;
qp->qplib_qp.rq.max_wqe = 0;
} else {
/* Allocate 1 more than what's provided so posting max doesn't
* mean empty
......@@ -1289,6 +1296,237 @@ static enum ib_mtu __to_ib_mtu(u32 mtu)
}
}
/* Shared Receive Queues */
int bnxt_re_destroy_srq(struct ib_srq *ib_srq)
{
struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
ib_srq);
struct bnxt_re_dev *rdev = srq->rdev;
struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
struct bnxt_qplib_nq *nq = NULL;
int rc;
if (qplib_srq->cq)
nq = qplib_srq->cq->nq;
rc = bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
if (rc) {
dev_err(rdev_to_dev(rdev), "Destroy HW SRQ failed!");
return rc;
}
if (srq->umem && !IS_ERR(srq->umem))
ib_umem_release(srq->umem);
kfree(srq);
atomic_dec(&rdev->srq_count);
if (nq)
nq->budget--;
return 0;
}
static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
struct bnxt_re_pd *pd,
struct bnxt_re_srq *srq,
struct ib_udata *udata)
{
struct bnxt_re_srq_req ureq;
struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
struct ib_umem *umem;
int bytes = 0;
struct ib_ucontext *context = pd->ib_pd.uobject->context;
struct bnxt_re_ucontext *cntx = container_of(context,
struct bnxt_re_ucontext,
ib_uctx);
if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
return -EFAULT;
bytes = (qplib_srq->max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
bytes = PAGE_ALIGN(bytes);
umem = ib_umem_get(context, ureq.srqva, bytes,
IB_ACCESS_LOCAL_WRITE, 1);
if (IS_ERR(umem))
return PTR_ERR(umem);
srq->umem = umem;
qplib_srq->nmap = umem->nmap;
qplib_srq->sglist = umem->sg_head.sgl;
qplib_srq->srq_handle = ureq.srq_handle;
qplib_srq->dpi = &cntx->dpi;
return 0;
}
struct ib_srq *bnxt_re_create_srq(struct ib_pd *ib_pd,
struct ib_srq_init_attr *srq_init_attr,
struct ib_udata *udata)
{
struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
struct bnxt_re_dev *rdev = pd->rdev;
struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
struct bnxt_re_srq *srq;
struct bnxt_qplib_nq *nq = NULL;
int rc, entries;
if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) {
dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded");
rc = -EINVAL;
goto exit;
}
if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
rc = -ENOTSUPP;
goto exit;
}
srq = kzalloc(sizeof(*srq), GFP_KERNEL);
if (!srq) {
rc = -ENOMEM;
goto exit;
}
srq->rdev = rdev;
srq->qplib_srq.pd = &pd->qplib_pd;
srq->qplib_srq.dpi = &rdev->dpi_privileged;
/* Allocate 1 more than what's provided so posting max doesn't
* mean empty
*/
entries = roundup_pow_of_two(srq_init_attr->attr.max_wr + 1);
if (entries > dev_attr->max_srq_wqes + 1)
entries = dev_attr->max_srq_wqes + 1;
srq->qplib_srq.max_wqe = entries;
srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;
srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
srq->srq_limit = srq_init_attr->attr.srq_limit;
srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id;
nq = &rdev->nq[0];
if (udata) {
rc = bnxt_re_init_user_srq(rdev, pd, srq, udata);
if (rc)
goto fail;
}
rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq);
if (rc) {
dev_err(rdev_to_dev(rdev), "Create HW SRQ failed!");
goto fail;
}
if (udata) {
struct bnxt_re_srq_resp resp;
resp.srqid = srq->qplib_srq.id;
rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
if (rc) {
dev_err(rdev_to_dev(rdev), "SRQ copy to udata failed!");
bnxt_qplib_destroy_srq(&rdev->qplib_res,
&srq->qplib_srq);
goto exit;
}
}
if (nq)
nq->budget++;
atomic_inc(&rdev->srq_count);
return &srq->ib_srq;
fail:
if (udata && srq->umem && !IS_ERR(srq->umem)) {
ib_umem_release(srq->umem);
srq->umem = NULL;
}
kfree(srq);
exit:
return ERR_PTR(rc);
}
int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
enum ib_srq_attr_mask srq_attr_mask,
struct ib_udata *udata)
{
struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
ib_srq);
struct bnxt_re_dev *rdev = srq->rdev;
int rc;
switch (srq_attr_mask) {
case IB_SRQ_MAX_WR:
/* SRQ resize is not supported */
break;
case IB_SRQ_LIMIT:
/* Change the SRQ threshold */
if (srq_attr->srq_limit > srq->qplib_srq.max_wqe)
return -EINVAL;
srq->qplib_srq.threshold = srq_attr->srq_limit;
rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq);
if (rc) {
dev_err(rdev_to_dev(rdev), "Modify HW SRQ failed!");
return rc;
}
/* On success, update the shadow */
srq->srq_limit = srq_attr->srq_limit;
/* No need to Build and send response back to udata */
break;
default:
dev_err(rdev_to_dev(rdev),
"Unsupported srq_attr_mask 0x%x", srq_attr_mask);
return -EINVAL;
}
return 0;
}
int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr)
{
struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
ib_srq);
struct bnxt_re_srq tsrq;
struct bnxt_re_dev *rdev = srq->rdev;
int rc;
/* Get live SRQ attr */
tsrq.qplib_srq.id = srq->qplib_srq.id;
rc = bnxt_qplib_query_srq(&rdev->qplib_res, &tsrq.qplib_srq);
if (rc) {
dev_err(rdev_to_dev(rdev), "Query HW SRQ failed!");
return rc;
}
srq_attr->max_wr = srq->qplib_srq.max_wqe;
srq_attr->max_sge = srq->qplib_srq.max_sge;
srq_attr->srq_limit = tsrq.qplib_srq.threshold;
return 0;
}
int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, struct ib_recv_wr *wr,
struct ib_recv_wr **bad_wr)
{
struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
ib_srq);
struct bnxt_qplib_swqe wqe;
unsigned long flags;
int rc = 0, payload_sz = 0;
spin_lock_irqsave(&srq->lock, flags);
while (wr) {
/* Transcribe each ib_recv_wr to qplib_swqe */
wqe.num_sge = wr->num_sge;
payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list,
wr->num_sge);
wqe.wr_id = wr->wr_id;
wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe);
if (rc) {
*bad_wr = wr;
break;
}
wr = wr->next;
}
spin_unlock_irqrestore(&srq->lock, flags);
return rc;
}
static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
struct bnxt_re_qp *qp1_qp,
int qp_attr_mask)
......
......@@ -68,6 +68,15 @@ struct bnxt_re_ah {
struct bnxt_qplib_ah qplib_ah;
};
struct bnxt_re_srq {
struct bnxt_re_dev *rdev;
u32 srq_limit;
struct ib_srq ib_srq;
struct bnxt_qplib_srq qplib_srq;
struct ib_umem *umem;
spinlock_t lock; /* protect srq */
};
struct bnxt_re_qp {
struct list_head list;
struct bnxt_re_dev *rdev;
......@@ -165,6 +174,16 @@ struct ib_ah *bnxt_re_create_ah(struct ib_pd *pd,
int bnxt_re_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
int bnxt_re_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
int bnxt_re_destroy_ah(struct ib_ah *ah);
struct ib_srq *bnxt_re_create_srq(struct ib_pd *pd,
struct ib_srq_init_attr *srq_init_attr,
struct ib_udata *udata);
int bnxt_re_modify_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr,
enum ib_srq_attr_mask srq_attr_mask,
struct ib_udata *udata);
int bnxt_re_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
int bnxt_re_destroy_srq(struct ib_srq *srq);
int bnxt_re_post_srq_recv(struct ib_srq *srq, struct ib_recv_wr *recv_wr,
struct ib_recv_wr **bad_recv_wr);
struct ib_qp *bnxt_re_create_qp(struct ib_pd *pd,
struct ib_qp_init_attr *qp_init_attr,
struct ib_udata *udata);
......
......@@ -588,6 +588,12 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
ibdev->query_ah = bnxt_re_query_ah;
ibdev->destroy_ah = bnxt_re_destroy_ah;
ibdev->create_srq = bnxt_re_create_srq;
ibdev->modify_srq = bnxt_re_modify_srq;
ibdev->query_srq = bnxt_re_query_srq;
ibdev->destroy_srq = bnxt_re_destroy_srq;
ibdev->post_srq_recv = bnxt_re_post_srq_recv;
ibdev->create_qp = bnxt_re_create_qp;
ibdev->modify_qp = bnxt_re_modify_qp;
ibdev->query_qp = bnxt_re_query_qp;
......@@ -689,10 +695,10 @@ static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev,
return rdev;
}
static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw,
struct creq_func_event *aeqe)
static int bnxt_re_handle_unaffi_async_event(struct creq_func_event
*unaffi_async)
{
switch (aeqe->event) {
switch (unaffi_async->event) {
case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR:
break;
case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR:
......@@ -721,6 +727,93 @@ static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw,
return 0;
}
static int bnxt_re_handle_qp_async_event(struct creq_qp_event *qp_event,
struct bnxt_re_qp *qp)
{
struct ib_event event;
memset(&event, 0, sizeof(event));
if (qp->qplib_qp.srq) {
event.device = &qp->rdev->ibdev;
event.element.qp = &qp->ib_qp;
event.event = IB_EVENT_QP_LAST_WQE_REACHED;
}
if (event.device && qp->ib_qp.event_handler)
qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context);
return 0;
}
static int bnxt_re_handle_affi_async_event(struct creq_qp_event *affi_async,
void *obj)
{
int rc = 0;
u8 event;
if (!obj)
return rc; /* QP was already dead, still return success */
event = affi_async->event;
if (event == CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION) {
struct bnxt_qplib_qp *lib_qp = obj;
struct bnxt_re_qp *qp = container_of(lib_qp, struct bnxt_re_qp,
qplib_qp);
rc = bnxt_re_handle_qp_async_event(affi_async, qp);
}
return rc;
}
static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw,
void *aeqe, void *obj)
{
struct creq_qp_event *affi_async;
struct creq_func_event *unaffi_async;
u8 type;
int rc;
type = ((struct creq_base *)aeqe)->type;
if (type == CREQ_BASE_TYPE_FUNC_EVENT) {
unaffi_async = aeqe;
rc = bnxt_re_handle_unaffi_async_event(unaffi_async);
} else {
affi_async = aeqe;
rc = bnxt_re_handle_affi_async_event(affi_async, obj);
}
return rc;
}
static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq,
struct bnxt_qplib_srq *handle, u8 event)
{
struct bnxt_re_srq *srq = container_of(handle, struct bnxt_re_srq,
qplib_srq);
struct ib_event ib_event;
int rc = 0;
if (!srq) {
dev_err(NULL, "%s: SRQ is NULL, SRQN not handled",
ROCE_DRV_MODULE_NAME);
rc = -EINVAL;
goto done;
}
ib_event.device = &srq->rdev->ibdev;
ib_event.element.srq = &srq->ib_srq;
if (event == NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT)
ib_event.event = IB_EVENT_SRQ_LIMIT_REACHED;
else
ib_event.event = IB_EVENT_SRQ_ERR;
if (srq->ib_srq.event_handler) {
/* Lock event_handler? */
(*srq->ib_srq.event_handler)(&ib_event,
srq->ib_srq.srq_context);
}
done:
return rc;
}
static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq,
struct bnxt_qplib_cq *handle)
{
......@@ -763,7 +856,8 @@ static int bnxt_re_init_res(struct bnxt_re_dev *rdev)
rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nq[i - 1],
i - 1, rdev->msix_entries[i].vector,
rdev->msix_entries[i].db_offset,
&bnxt_re_cqn_handler, NULL);
&bnxt_re_cqn_handler,
&bnxt_re_srqn_handler);
if (rc) {
dev_err(rdev_to_dev(rdev),
......
......@@ -52,6 +52,7 @@
static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq);
static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp);
static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type);
static void bnxt_qplib_cancel_phantom_processing(struct bnxt_qplib_qp *qp)
{
......@@ -278,6 +279,7 @@ static void bnxt_qplib_service_nq(unsigned long data)
struct nq_base *nqe, **nq_ptr;
struct bnxt_qplib_cq *cq;
int num_cqne_processed = 0;
int num_srqne_processed = 0;
u32 sw_cons, raw_cons;
u16 type;
int budget = nq->budget;
......@@ -320,6 +322,26 @@ static void bnxt_qplib_service_nq(unsigned long data)
spin_unlock_bh(&cq->compl_lock);
break;
}
case NQ_BASE_TYPE_SRQ_EVENT:
{
struct nq_srq_event *nqsrqe =
(struct nq_srq_event *)nqe;
q_handle = le32_to_cpu(nqsrqe->srq_handle_low);
q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high)
<< 32;
bnxt_qplib_arm_srq((struct bnxt_qplib_srq *)q_handle,
DBR_DBR_TYPE_SRQ_ARMENA);
if (!nq->srqn_handler(nq,
(struct bnxt_qplib_srq *)q_handle,
nqsrqe->event))
num_srqne_processed++;
else
dev_warn(&nq->pdev->dev,
"QPLIB: SRQ event 0x%x not handled",
nqsrqe->event);
break;
}
case NQ_BASE_TYPE_DBQ_EVENT:
break;
default:
......@@ -384,17 +406,19 @@ int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
int (*cqn_handler)(struct bnxt_qplib_nq *nq,
struct bnxt_qplib_cq *),
int (*srqn_handler)(struct bnxt_qplib_nq *nq,
void *, u8 event))
struct bnxt_qplib_srq *,
u8 event))
{
resource_size_t nq_base;
int rc = -1;
nq->pdev = pdev;
nq->vector = msix_vector;
if (cqn_handler)
nq->cqn_handler = cqn_handler;
nq->cqn_handler = cqn_handler;
nq->srqn_handler = srqn_handler;
if (srqn_handler)
nq->srqn_handler = srqn_handler;
tasklet_init(&nq->worker, bnxt_qplib_service_nq, (unsigned long)nq);
......@@ -468,6 +492,238 @@ int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq)
return 0;
}
/* SRQ */
static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type)
{
struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
struct dbr_dbr db_msg = { 0 };
void __iomem *db;
u32 sw_prod = 0;
/* Ring DB */
sw_prod = (arm_type == DBR_DBR_TYPE_SRQ_ARM) ? srq->threshold :
HWQ_CMP(srq_hwq->prod, srq_hwq);
db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
DBR_DBR_INDEX_MASK);
db_msg.type_xid = cpu_to_le32(((srq->id << DBR_DBR_XID_SFT) &
DBR_DBR_XID_MASK) | arm_type);
db = (arm_type == DBR_DBR_TYPE_SRQ_ARMENA) ?
srq->dbr_base : srq->dpi->dbr;
wmb(); /* barrier before db ring */
__iowrite64_copy(db, &db_msg, sizeof(db_msg) / sizeof(u64));
}
int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
struct bnxt_qplib_srq *srq)
{
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
struct cmdq_destroy_srq req;
struct creq_destroy_srq_resp resp;
u16 cmd_flags = 0;
int rc;
RCFW_CMD_PREP(req, DESTROY_SRQ, cmd_flags);
/* Configure the request */
req.srq_cid = cpu_to_le32(srq->id);
rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
(void *)&resp, NULL, 0);
if (rc)
return rc;
bnxt_qplib_free_hwq(res->pdev, &srq->hwq);
kfree(srq->swq);
return 0;
}
int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
struct bnxt_qplib_srq *srq)
{
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
struct cmdq_create_srq req;
struct creq_create_srq_resp resp;
struct bnxt_qplib_pbl *pbl;
u16 cmd_flags = 0;
int rc, idx;
srq->hwq.max_elements = srq->max_wqe;
rc = bnxt_qplib_alloc_init_hwq(res->pdev, &srq->hwq, srq->sglist,
srq->nmap, &srq->hwq.max_elements,
BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
PAGE_SIZE, HWQ_TYPE_QUEUE);
if (rc)
goto exit;
srq->swq = kcalloc(srq->hwq.max_elements, sizeof(*srq->swq),
GFP_KERNEL);
if (!srq->swq)
goto fail;
RCFW_CMD_PREP(req, CREATE_SRQ, cmd_flags);
/* Configure the request */
req.dpi = cpu_to_le32(srq->dpi->dpi);
req.srq_handle = cpu_to_le64(srq);
req.srq_size = cpu_to_le16((u16)srq->hwq.max_elements);
pbl = &srq->hwq.pbl[PBL_LVL_0];
req.pg_size_lvl = cpu_to_le16((((u16)srq->hwq.level &
CMDQ_CREATE_SRQ_LVL_MASK) <<
CMDQ_CREATE_SRQ_LVL_SFT) |
(pbl->pg_size == ROCE_PG_SIZE_4K ?
CMDQ_CREATE_SRQ_PG_SIZE_PG_4K :
pbl->pg_size == ROCE_PG_SIZE_8K ?
CMDQ_CREATE_SRQ_PG_SIZE_PG_8K :
pbl->pg_size == ROCE_PG_SIZE_64K ?
CMDQ_CREATE_SRQ_PG_SIZE_PG_64K :
pbl->pg_size == ROCE_PG_SIZE_2M ?
CMDQ_CREATE_SRQ_PG_SIZE_PG_2M :
pbl->pg_size == ROCE_PG_SIZE_8M ?
CMDQ_CREATE_SRQ_PG_SIZE_PG_8M :
pbl->pg_size == ROCE_PG_SIZE_1G ?
CMDQ_CREATE_SRQ_PG_SIZE_PG_1G :
CMDQ_CREATE_SRQ_PG_SIZE_PG_4K));
req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
req.pd_id = cpu_to_le32(srq->pd->id);
req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id);
rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
(void *)&resp, NULL, 0);
if (rc)
goto fail;
spin_lock_init(&srq->lock);
srq->start_idx = 0;
srq->last_idx = srq->hwq.max_elements - 1;
for (idx = 0; idx < srq->hwq.max_elements; idx++)
srq