Commit b038ced7 authored by Steve Wise's avatar Steve Wise Committed by Roland Dreier
Browse files

RDMA/cxgb3: Add driver for Chelsio T3 RNIC



Add an RDMA/iWARP driver for the Chelsio T3 1GbE and 10GbE adapters.
Signed-off-by: default avatarSteve Wise <swise@opengridcomputing.com>
Signed-off-by: default avatarRoland Dreier <rolandd@cisco.com>
parent c7f743a6
......@@ -38,6 +38,7 @@ source "drivers/infiniband/hw/mthca/Kconfig"
source "drivers/infiniband/hw/ipath/Kconfig"
source "drivers/infiniband/hw/ehca/Kconfig"
source "drivers/infiniband/hw/amso1100/Kconfig"
source "drivers/infiniband/hw/cxgb3/Kconfig"
source "drivers/infiniband/ulp/ipoib/Kconfig"
......
......@@ -3,6 +3,7 @@ obj-$(CONFIG_INFINIBAND_MTHCA) += hw/mthca/
obj-$(CONFIG_INFINIBAND_IPATH) += hw/ipath/
obj-$(CONFIG_INFINIBAND_EHCA) += hw/ehca/
obj-$(CONFIG_INFINIBAND_AMSO1100) += hw/amso1100/
obj-$(CONFIG_INFINIBAND_CXGB3) += hw/cxgb3/
obj-$(CONFIG_INFINIBAND_IPOIB) += ulp/ipoib/
obj-$(CONFIG_INFINIBAND_SRP) += ulp/srp/
obj-$(CONFIG_INFINIBAND_ISER) += ulp/iser/
config INFINIBAND_CXGB3
tristate "Chelsio RDMA Driver"
depends on CHELSIO_T3 && INFINIBAND && INET
select GENERIC_ALLOCATOR
---help---
This is an iWARP/RDMA driver for the Chelsio T3 1GbE and
10GbE adapters.
For general information about Chelsio and our products, visit
our website at <http://www.chelsio.com>.
For customer support, please visit our customer support page at
<http://www.chelsio.com/support.htm>.
Please send feedback to <linux-bugs@chelsio.com>.
To compile this driver as a module, choose M here: the module
will be called iw_cxgb3.
config INFINIBAND_CXGB3_DEBUG
bool "Verbose debugging output"
depends on INFINIBAND_CXGB3
default n
---help---
This option causes the Chelsio RDMA driver to produce copious
amounts of debug messages. Select this if you are developing
the driver or trying to diagnose a problem.
EXTRA_CFLAGS += -I$(TOPDIR)/drivers/net/cxgb3 \
-I$(TOPDIR)/drivers/infiniband/hw/cxgb3/core
obj-$(CONFIG_INFINIBAND_CXGB3) += iw_cxgb3.o
iw_cxgb3-y := iwch_cm.o iwch_ev.o iwch_cq.o iwch_qp.o iwch_mem.o \
iwch_provider.o iwch.o cxio_hal.o cxio_resource.o
ifdef CONFIG_INFINIBAND_CXGB3_DEBUG
EXTRA_CFLAGS += -DDEBUG
iw_cxgb3-y += cxio_dbg.o
endif
/*
* Copyright (c) 2006 Chelsio, Inc. All rights reserved.
* Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifdef DEBUG
#include <linux/types.h>
#include "common.h"
#include "cxgb3_ioctl.h"
#include "cxio_hal.h"
#include "cxio_wr.h"
void cxio_dump_tpt(struct cxio_rdev *rdev, u32 stag)
{
struct ch_mem_range *m;
u64 *data;
int rc;
int size = 32;
m = kmalloc(sizeof(*m) + size, GFP_ATOMIC);
if (!m) {
PDBG("%s couldn't allocate memory.\n", __FUNCTION__);
return;
}
m->mem_id = MEM_PMRX;
m->addr = (stag>>8) * 32 + rdev->rnic_info.tpt_base;
m->len = size;
PDBG("%s TPT addr 0x%x len %d\n", __FUNCTION__, m->addr, m->len);
rc = rdev->t3cdev_p->ctl(rdev->t3cdev_p, RDMA_GET_MEM, m);
if (rc) {
PDBG("%s toectl returned error %d\n", __FUNCTION__, rc);
kfree(m);
return;
}
data = (u64 *)m->buf;
while (size > 0) {
PDBG("TPT %08x: %016llx\n", m->addr, (unsigned long long) *data);
size -= 8;
data++;
m->addr += 8;
}
kfree(m);
}
void cxio_dump_pbl(struct cxio_rdev *rdev, u32 pbl_addr, uint len, u8 shift)
{
struct ch_mem_range *m;
u64 *data;
int rc;
int size, npages;
shift += 12;
npages = (len + (1ULL << shift) - 1) >> shift;
size = npages * sizeof(u64);
m = kmalloc(sizeof(*m) + size, GFP_ATOMIC);
if (!m) {
PDBG("%s couldn't allocate memory.\n", __FUNCTION__);
return;
}
m->mem_id = MEM_PMRX;
m->addr = pbl_addr;
m->len = size;
PDBG("%s PBL addr 0x%x len %d depth %d\n",
__FUNCTION__, m->addr, m->len, npages);
rc = rdev->t3cdev_p->ctl(rdev->t3cdev_p, RDMA_GET_MEM, m);
if (rc) {
PDBG("%s toectl returned error %d\n", __FUNCTION__, rc);
kfree(m);
return;
}
data = (u64 *)m->buf;
while (size > 0) {
PDBG("PBL %08x: %016llx\n", m->addr, (unsigned long long) *data);
size -= 8;
data++;
m->addr += 8;
}
kfree(m);
}
void cxio_dump_wqe(union t3_wr *wqe)
{
__be64 *data = (__be64 *)wqe;
uint size = (uint)(be64_to_cpu(*data) & 0xff);
if (size == 0)
size = 8;
while (size > 0) {
PDBG("WQE %p: %016llx\n", data,
(unsigned long long) be64_to_cpu(*data));
size--;
data++;
}
}
void cxio_dump_wce(struct t3_cqe *wce)
{
__be64 *data = (__be64 *)wce;
int size = sizeof(*wce);
while (size > 0) {
PDBG("WCE %p: %016llx\n", data,
(unsigned long long) be64_to_cpu(*data));
size -= 8;
data++;
}
}
void cxio_dump_rqt(struct cxio_rdev *rdev, u32 hwtid, int nents)
{
struct ch_mem_range *m;
int size = nents * 64;
u64 *data;
int rc;
m = kmalloc(sizeof(*m) + size, GFP_ATOMIC);
if (!m) {
PDBG("%s couldn't allocate memory.\n", __FUNCTION__);
return;
}
m->mem_id = MEM_PMRX;
m->addr = ((hwtid)<<10) + rdev->rnic_info.rqt_base;
m->len = size;
PDBG("%s RQT addr 0x%x len %d\n", __FUNCTION__, m->addr, m->len);
rc = rdev->t3cdev_p->ctl(rdev->t3cdev_p, RDMA_GET_MEM, m);
if (rc) {
PDBG("%s toectl returned error %d\n", __FUNCTION__, rc);
kfree(m);
return;
}
data = (u64 *)m->buf;
while (size > 0) {
PDBG("RQT %08x: %016llx\n", m->addr, (unsigned long long) *data);
size -= 8;
data++;
m->addr += 8;
}
kfree(m);
}
void cxio_dump_tcb(struct cxio_rdev *rdev, u32 hwtid)
{
struct ch_mem_range *m;
int size = TCB_SIZE;
u32 *data;
int rc;
m = kmalloc(sizeof(*m) + size, GFP_ATOMIC);
if (!m) {
PDBG("%s couldn't allocate memory.\n", __FUNCTION__);
return;
}
m->mem_id = MEM_CM;
m->addr = hwtid * size;
m->len = size;
PDBG("%s TCB %d len %d\n", __FUNCTION__, m->addr, m->len);
rc = rdev->t3cdev_p->ctl(rdev->t3cdev_p, RDMA_GET_MEM, m);
if (rc) {
PDBG("%s toectl returned error %d\n", __FUNCTION__, rc);
kfree(m);
return;
}
data = (u32 *)m->buf;
while (size > 0) {
printk("%2u: %08x %08x %08x %08x %08x %08x %08x %08x\n",
m->addr,
*(data+2), *(data+3), *(data),*(data+1),
*(data+6), *(data+7), *(data+4), *(data+5));
size -= 32;
data += 8;
m->addr += 32;
}
kfree(m);
}
#endif
This diff is collapsed.
/*
* Copyright (c) 2006 Chelsio, Inc. All rights reserved.
* Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef __CXIO_HAL_H__
#define __CXIO_HAL_H__
#include <linux/list.h>
#include <linux/mutex.h>
#include "t3_cpl.h"
#include "t3cdev.h"
#include "cxgb3_ctl_defs.h"
#include "cxio_wr.h"
#define T3_CTRL_QP_ID FW_RI_SGEEC_START
#define T3_CTL_QP_TID FW_RI_TID_START
#define T3_CTRL_QP_SIZE_LOG2 8
#define T3_CTRL_CQ_ID 0
/* TBD */
#define T3_MAX_NUM_RI (1<<15)
#define T3_MAX_NUM_QP (1<<15)
#define T3_MAX_NUM_CQ (1<<15)
#define T3_MAX_NUM_PD (1<<15)
#define T3_MAX_PBL_SIZE 256
#define T3_MAX_RQ_SIZE 1024
#define T3_MAX_NUM_STAG (1<<15)
#define T3_STAG_UNSET 0xffffffff
#define T3_MAX_DEV_NAME_LEN 32
struct cxio_hal_ctrl_qp {
u32 wptr;
u32 rptr;
struct mutex lock; /* for the wtpr, can sleep */
wait_queue_head_t waitq;/* wait for RspQ/CQE msg */
union t3_wr *workq; /* the work request queue */
dma_addr_t dma_addr; /* pci bus address of the workq */
DECLARE_PCI_UNMAP_ADDR(mapping)
void __iomem *doorbell;
};
struct cxio_hal_resource {
struct kfifo *tpt_fifo;
spinlock_t tpt_fifo_lock;
struct kfifo *qpid_fifo;
spinlock_t qpid_fifo_lock;
struct kfifo *cqid_fifo;
spinlock_t cqid_fifo_lock;
struct kfifo *pdid_fifo;
spinlock_t pdid_fifo_lock;
};
struct cxio_qpid_list {
struct list_head entry;
u32 qpid;
};
struct cxio_ucontext {
struct list_head qpids;
struct mutex lock;
};
struct cxio_rdev {
char dev_name[T3_MAX_DEV_NAME_LEN];
struct t3cdev *t3cdev_p;
struct rdma_info rnic_info;
struct adap_ports port_info;
struct cxio_hal_resource *rscp;
struct cxio_hal_ctrl_qp ctrl_qp;
void *ulp;
unsigned long qpshift;
u32 qpnr;
u32 qpmask;
struct cxio_ucontext uctx;
struct gen_pool *pbl_pool;
struct gen_pool *rqt_pool;
struct list_head entry;
};
static inline int cxio_num_stags(struct cxio_rdev *rdev_p)
{
return min((int)T3_MAX_NUM_STAG, (int)((rdev_p->rnic_info.tpt_top - rdev_p->rnic_info.tpt_base) >> 5));
}
typedef void (*cxio_hal_ev_callback_func_t) (struct cxio_rdev * rdev_p,
struct sk_buff * skb);
#define RSPQ_CQID(rsp) (be32_to_cpu(rsp->cq_ptrid) & 0xffff)
#define RSPQ_CQPTR(rsp) ((be32_to_cpu(rsp->cq_ptrid) >> 16) & 0xffff)
#define RSPQ_GENBIT(rsp) ((be32_to_cpu(rsp->flags) >> 16) & 1)
#define RSPQ_OVERFLOW(rsp) ((be32_to_cpu(rsp->flags) >> 17) & 1)
#define RSPQ_AN(rsp) ((be32_to_cpu(rsp->flags) >> 18) & 1)
#define RSPQ_SE(rsp) ((be32_to_cpu(rsp->flags) >> 19) & 1)
#define RSPQ_NOTIFY(rsp) ((be32_to_cpu(rsp->flags) >> 20) & 1)
#define RSPQ_CQBRANCH(rsp) ((be32_to_cpu(rsp->flags) >> 21) & 1)
#define RSPQ_CREDIT_THRESH(rsp) ((be32_to_cpu(rsp->flags) >> 22) & 1)
struct respQ_msg_t {
__be32 flags; /* flit 0 */
__be32 cq_ptrid;
__be64 rsvd; /* flit 1 */
struct t3_cqe cqe; /* flits 2-3 */
};
enum t3_cq_opcode {
CQ_ARM_AN = 0x2,
CQ_ARM_SE = 0x6,
CQ_FORCE_AN = 0x3,
CQ_CREDIT_UPDATE = 0x7
};
int cxio_rdev_open(struct cxio_rdev *rdev);
void cxio_rdev_close(struct cxio_rdev *rdev);
int cxio_hal_cq_op(struct cxio_rdev *rdev, struct t3_cq *cq,
enum t3_cq_opcode op, u32 credit);
int cxio_hal_clear_qp_ctx(struct cxio_rdev *rdev, u32 qpid);
int cxio_create_cq(struct cxio_rdev *rdev, struct t3_cq *cq);
int cxio_destroy_cq(struct cxio_rdev *rdev, struct t3_cq *cq);
int cxio_resize_cq(struct cxio_rdev *rdev, struct t3_cq *cq);
void cxio_release_ucontext(struct cxio_rdev *rdev, struct cxio_ucontext *uctx);
void cxio_init_ucontext(struct cxio_rdev *rdev, struct cxio_ucontext *uctx);
int cxio_create_qp(struct cxio_rdev *rdev, u32 kernel_domain, struct t3_wq *wq,
struct cxio_ucontext *uctx);
int cxio_destroy_qp(struct cxio_rdev *rdev, struct t3_wq *wq,
struct cxio_ucontext *uctx);
int cxio_peek_cq(struct t3_wq *wr, struct t3_cq *cq, int opcode);
int cxio_allocate_stag(struct cxio_rdev *rdev, u32 * stag, u32 pdid,
enum tpt_mem_perm perm, u32 * pbl_size, u32 * pbl_addr);
int cxio_register_phys_mem(struct cxio_rdev *rdev, u32 * stag, u32 pdid,
enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,
u8 page_size, __be64 *pbl, u32 *pbl_size,
u32 *pbl_addr);
int cxio_reregister_phys_mem(struct cxio_rdev *rdev, u32 * stag, u32 pdid,
enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,
u8 page_size, __be64 *pbl, u32 *pbl_size,
u32 *pbl_addr);
int cxio_dereg_mem(struct cxio_rdev *rdev, u32 stag, u32 pbl_size,
u32 pbl_addr);
int cxio_allocate_window(struct cxio_rdev *rdev, u32 * stag, u32 pdid);
int cxio_deallocate_window(struct cxio_rdev *rdev, u32 stag);
int cxio_rdma_init(struct cxio_rdev *rdev, struct t3_rdma_init_attr *attr);
void cxio_register_ev_cb(cxio_hal_ev_callback_func_t ev_cb);
void cxio_unregister_ev_cb(cxio_hal_ev_callback_func_t ev_cb);
u32 cxio_hal_get_rhdl(void);
void cxio_hal_put_rhdl(u32 rhdl);
u32 cxio_hal_get_pdid(struct cxio_hal_resource *rscp);
void cxio_hal_put_pdid(struct cxio_hal_resource *rscp, u32 pdid);
int __init cxio_hal_init(void);
void __exit cxio_hal_exit(void);
void cxio_flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count);
void cxio_flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count);
void cxio_count_rcqes(struct t3_cq *cq, struct t3_wq *wq, int *count);
void cxio_count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count);
void cxio_flush_hw_cq(struct t3_cq *cq);
int cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq, struct t3_cqe *cqe,
u8 *cqe_flushed, u64 *cookie, u32 *credit);
#define MOD "iw_cxgb3: "
#define PDBG(fmt, args...) pr_debug(MOD fmt, ## args)
#ifdef DEBUG
void cxio_dump_tpt(struct cxio_rdev *rev, u32 stag);
void cxio_dump_pbl(struct cxio_rdev *rev, u32 pbl_addr, uint len, u8 shift);
void cxio_dump_wqe(union t3_wr *wqe);
void cxio_dump_wce(struct t3_cqe *wce);
void cxio_dump_rqt(struct cxio_rdev *rdev, u32 hwtid, int nents);
void cxio_dump_tcb(struct cxio_rdev *rdev, u32 hwtid);
#endif
#endif
/*
* Copyright (c) 2006 Chelsio, Inc. All rights reserved.
* Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/* Crude resource management */
#include <linux/kernel.h>
#include <linux/random.h>
#include <linux/slab.h>
#include <linux/kfifo.h>
#include <linux/spinlock.h>
#include <linux/errno.h>
#include "cxio_resource.h"
#include "cxio_hal.h"
static struct kfifo *rhdl_fifo;
static spinlock_t rhdl_fifo_lock;
#define RANDOM_SIZE 16
static int __cxio_init_resource_fifo(struct kfifo **fifo,
spinlock_t *fifo_lock,
u32 nr, u32 skip_low,
u32 skip_high,
int random)
{
u32 i, j, entry = 0, idx;
u32 random_bytes;
u32 rarray[16];
spin_lock_init(fifo_lock);
*fifo = kfifo_alloc(nr * sizeof(u32), GFP_KERNEL, fifo_lock);
if (IS_ERR(*fifo))
return -ENOMEM;
for (i = 0; i < skip_low + skip_high; i++)
__kfifo_put(*fifo, (unsigned char *) &entry, sizeof(u32));
if (random) {
j = 0;
random_bytes = random32();
for (i = 0; i < RANDOM_SIZE; i++)
rarray[i] = i + skip_low;
for (i = skip_low + RANDOM_SIZE; i < nr - skip_high; i++) {
if (j >= RANDOM_SIZE) {
j = 0;
random_bytes = random32();
}
idx = (random_bytes >> (j * 2)) & 0xF;
__kfifo_put(*fifo,
(unsigned char *) &rarray[idx],
sizeof(u32));
rarray[idx] = i;
j++;
}
for (i = 0; i < RANDOM_SIZE; i++)
__kfifo_put(*fifo,
(unsigned char *) &rarray[i],
sizeof(u32));
} else
for (i = skip_low; i < nr - skip_high; i++)
__kfifo_put(*fifo, (unsigned char *) &i, sizeof(u32));
for (i = 0; i < skip_low + skip_high; i++)
kfifo_get(*fifo, (unsigned char *) &entry, sizeof(u32));
return 0;
}
static int cxio_init_resource_fifo(struct kfifo **fifo, spinlock_t * fifo_lock,
u32 nr, u32 skip_low, u32 skip_high)
{
return (__cxio_init_resource_fifo(fifo, fifo_lock, nr, skip_low,
skip_high, 0));
}
static int cxio_init_resource_fifo_random(struct kfifo **fifo,
spinlock_t * fifo_lock,
u32 nr, u32 skip_low, u32 skip_high)
{
return (__cxio_init_resource_fifo(fifo, fifo_lock, nr, skip_low,
skip_high, 1));
}
static int cxio_init_qpid_fifo(struct cxio_rdev *rdev_p)
{
u32 i;
spin_lock_init(&rdev_p->rscp->qpid_fifo_lock);
rdev_p->rscp->qpid_fifo = kfifo_alloc(T3_MAX_NUM_QP * sizeof(u32),
GFP_KERNEL,
&rdev_p->rscp->qpid_fifo_lock);
if (IS_ERR(rdev_p->rscp->qpid_fifo))
return -ENOMEM;
for (i = 16; i < T3_MAX_NUM_QP; i++)
if (!(i & rdev_p->qpmask))
__kfifo_put(rdev_p->rscp->qpid_fifo,
(unsigned char *) &i, sizeof(u32));
return 0;
}
int cxio_hal_init_rhdl_resource(u32 nr_rhdl)
{
return cxio_init_resource_fifo(&rhdl_fifo, &rhdl_fifo_lock, nr_rhdl, 1,
0);
}