verbs.c 34 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
6
/*
 * Copyright (c) 2004 Mellanox Technologies Ltd.  All rights reserved.
 * Copyright (c) 2004 Infinicon Corporation.  All rights reserved.
 * Copyright (c) 2004 Intel Corporation.  All rights reserved.
 * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
 * Copyright (c) 2004 Voltaire Corporation.  All rights reserved.
7
 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
8
 * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
Linus Torvalds's avatar
Linus Torvalds committed
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/errno.h>
#include <linux/err.h>
41
#include <linux/export.h>
42
#include <linux/string.h>
43
#include <linux/slab.h>
Linus Torvalds's avatar
Linus Torvalds committed
44

45
46
#include <rdma/ib_verbs.h>
#include <rdma/ib_cache.h>
47
#include <rdma/ib_addr.h>
Linus Torvalds's avatar
Linus Torvalds committed
48

49
#include "core_priv.h"
Linus Torvalds's avatar
Linus Torvalds committed
50

51
__attribute_const__ int ib_rate_to_mult(enum ib_rate rate)
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
{
	switch (rate) {
	case IB_RATE_2_5_GBPS: return  1;
	case IB_RATE_5_GBPS:   return  2;
	case IB_RATE_10_GBPS:  return  4;
	case IB_RATE_20_GBPS:  return  8;
	case IB_RATE_30_GBPS:  return 12;
	case IB_RATE_40_GBPS:  return 16;
	case IB_RATE_60_GBPS:  return 24;
	case IB_RATE_80_GBPS:  return 32;
	case IB_RATE_120_GBPS: return 48;
	default:	       return -1;
	}
}
EXPORT_SYMBOL(ib_rate_to_mult);

68
__attribute_const__ enum ib_rate mult_to_ib_rate(int mult)
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
{
	switch (mult) {
	case 1:  return IB_RATE_2_5_GBPS;
	case 2:  return IB_RATE_5_GBPS;
	case 4:  return IB_RATE_10_GBPS;
	case 8:  return IB_RATE_20_GBPS;
	case 12: return IB_RATE_30_GBPS;
	case 16: return IB_RATE_40_GBPS;
	case 24: return IB_RATE_60_GBPS;
	case 32: return IB_RATE_80_GBPS;
	case 48: return IB_RATE_120_GBPS;
	default: return IB_RATE_PORT_CURRENT;
	}
}
EXPORT_SYMBOL(mult_to_ib_rate);

85
__attribute_const__ int ib_rate_to_mbps(enum ib_rate rate)
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
{
	switch (rate) {
	case IB_RATE_2_5_GBPS: return 2500;
	case IB_RATE_5_GBPS:   return 5000;
	case IB_RATE_10_GBPS:  return 10000;
	case IB_RATE_20_GBPS:  return 20000;
	case IB_RATE_30_GBPS:  return 30000;
	case IB_RATE_40_GBPS:  return 40000;
	case IB_RATE_60_GBPS:  return 60000;
	case IB_RATE_80_GBPS:  return 80000;
	case IB_RATE_120_GBPS: return 120000;
	case IB_RATE_14_GBPS:  return 14062;
	case IB_RATE_56_GBPS:  return 56250;
	case IB_RATE_112_GBPS: return 112500;
	case IB_RATE_168_GBPS: return 168750;
	case IB_RATE_25_GBPS:  return 25781;
	case IB_RATE_100_GBPS: return 103125;
	case IB_RATE_200_GBPS: return 206250;
	case IB_RATE_300_GBPS: return 309375;
	default:	       return -1;
	}
}
EXPORT_SYMBOL(ib_rate_to_mbps);

110
__attribute_const__ enum rdma_transport_type
Tom Tucker's avatar
Tom Tucker committed
111
112
113
114
115
116
117
118
119
rdma_node_get_transport(enum rdma_node_type node_type)
{
	switch (node_type) {
	case RDMA_NODE_IB_CA:
	case RDMA_NODE_IB_SWITCH:
	case RDMA_NODE_IB_ROUTER:
		return RDMA_TRANSPORT_IB;
	case RDMA_NODE_RNIC:
		return RDMA_TRANSPORT_IWARP;
120
	case RDMA_NODE_USNIC:
121
122
		return RDMA_TRANSPORT_USNIC;
	case RDMA_NODE_USNIC_UDP:
123
		return RDMA_TRANSPORT_USNIC_UDP;
Tom Tucker's avatar
Tom Tucker committed
124
125
126
127
128
129
130
	default:
		BUG();
		return 0;
	}
}
EXPORT_SYMBOL(rdma_node_get_transport);

131
132
133
134
135
136
137
138
139
enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num)
{
	if (device->get_link_layer)
		return device->get_link_layer(device, port_num);

	switch (rdma_node_get_transport(device->node_type)) {
	case RDMA_TRANSPORT_IB:
		return IB_LINK_LAYER_INFINIBAND;
	case RDMA_TRANSPORT_IWARP:
140
	case RDMA_TRANSPORT_USNIC:
141
	case RDMA_TRANSPORT_USNIC_UDP:
142
143
144
145
146
147
148
		return IB_LINK_LAYER_ETHERNET;
	default:
		return IB_LINK_LAYER_UNSPECIFIED;
	}
}
EXPORT_SYMBOL(rdma_port_get_link_layer);

Linus Torvalds's avatar
Linus Torvalds committed
149
150
151
152
153
154
/* Protection domains */

struct ib_pd *ib_alloc_pd(struct ib_device *device)
{
	struct ib_pd *pd;

155
	pd = device->alloc_pd(device, NULL, NULL);
Linus Torvalds's avatar
Linus Torvalds committed
156
157

	if (!IS_ERR(pd)) {
158
159
		pd->device  = device;
		pd->uobject = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
		atomic_set(&pd->usecnt, 0);
	}

	return pd;
}
EXPORT_SYMBOL(ib_alloc_pd);

int ib_dealloc_pd(struct ib_pd *pd)
{
	if (atomic_read(&pd->usecnt))
		return -EBUSY;

	return pd->device->dealloc_pd(pd);
}
EXPORT_SYMBOL(ib_dealloc_pd);

/* Address handles */

struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
{
	struct ib_ah *ah;

	ah = pd->device->create_ah(pd, ah_attr);

	if (!IS_ERR(ah)) {
185
186
187
		ah->device  = pd->device;
		ah->pd      = pd;
		ah->uobject = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
188
189
190
191
192
193
194
		atomic_inc(&pd->usecnt);
	}

	return ah;
}
EXPORT_SYMBOL(ib_create_ah);

Sean Hefty's avatar
Sean Hefty committed
195
196
int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, struct ib_wc *wc,
		       struct ib_grh *grh, struct ib_ah_attr *ah_attr)
197
198
199
200
201
{
	u32 flow_class;
	u16 gid_index;
	int ret;

Sean Hefty's avatar
Sean Hefty committed
202
	memset(ah_attr, 0, sizeof *ah_attr);
203
	if (rdma_cap_eth_ah(device, port_num)) {
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
		if (!(wc->wc_flags & IB_WC_GRH))
			return -EPROTOTYPE;

		if (wc->wc_flags & IB_WC_WITH_SMAC &&
		    wc->wc_flags & IB_WC_WITH_VLAN) {
			memcpy(ah_attr->dmac, wc->smac, ETH_ALEN);
			ah_attr->vlan_id = wc->vlan_id;
		} else {
			ret = rdma_addr_find_dmac_by_grh(&grh->dgid, &grh->sgid,
					ah_attr->dmac, &ah_attr->vlan_id);
			if (ret)
				return ret;
		}
	} else {
		ah_attr->vlan_id = 0xffff;
	}

Sean Hefty's avatar
Sean Hefty committed
221
222
223
224
	ah_attr->dlid = wc->slid;
	ah_attr->sl = wc->sl;
	ah_attr->src_path_bits = wc->dlid_path_bits;
	ah_attr->port_num = port_num;
225
226

	if (wc->wc_flags & IB_WC_GRH) {
Sean Hefty's avatar
Sean Hefty committed
227
228
		ah_attr->ah_flags = IB_AH_GRH;
		ah_attr->grh.dgid = grh->sgid;
229

Sean Hefty's avatar
Sean Hefty committed
230
		ret = ib_find_cached_gid(device, &grh->dgid, &port_num,
231
232
					 &gid_index);
		if (ret)
Sean Hefty's avatar
Sean Hefty committed
233
			return ret;
234

Sean Hefty's avatar
Sean Hefty committed
235
		ah_attr->grh.sgid_index = (u8) gid_index;
236
		flow_class = be32_to_cpu(grh->version_tclass_flow);
Sean Hefty's avatar
Sean Hefty committed
237
		ah_attr->grh.flow_label = flow_class & 0xFFFFF;
238
		ah_attr->grh.hop_limit = 0xFF;
Sean Hefty's avatar
Sean Hefty committed
239
		ah_attr->grh.traffic_class = (flow_class >> 20) & 0xFF;
240
	}
Sean Hefty's avatar
Sean Hefty committed
241
242
243
244
245
246
247
248
249
250
251
252
253
	return 0;
}
EXPORT_SYMBOL(ib_init_ah_from_wc);

struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, struct ib_wc *wc,
				   struct ib_grh *grh, u8 port_num)
{
	struct ib_ah_attr ah_attr;
	int ret;

	ret = ib_init_ah_from_wc(pd->device, port_num, wc, grh, &ah_attr);
	if (ret)
		return ERR_PTR(ret);
254
255
256
257
258

	return ib_create_ah(pd, &ah_attr);
}
EXPORT_SYMBOL(ib_create_ah_from_wc);

Linus Torvalds's avatar
Linus Torvalds committed
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
int ib_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
{
	return ah->device->modify_ah ?
		ah->device->modify_ah(ah, ah_attr) :
		-ENOSYS;
}
EXPORT_SYMBOL(ib_modify_ah);

int ib_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
{
	return ah->device->query_ah ?
		ah->device->query_ah(ah, ah_attr) :
		-ENOSYS;
}
EXPORT_SYMBOL(ib_query_ah);

int ib_destroy_ah(struct ib_ah *ah)
{
	struct ib_pd *pd;
	int ret;

	pd = ah->pd;
	ret = ah->device->destroy_ah(ah);
	if (!ret)
		atomic_dec(&pd->usecnt);

	return ret;
}
EXPORT_SYMBOL(ib_destroy_ah);

289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
/* Shared receive queues */

struct ib_srq *ib_create_srq(struct ib_pd *pd,
			     struct ib_srq_init_attr *srq_init_attr)
{
	struct ib_srq *srq;

	if (!pd->device->create_srq)
		return ERR_PTR(-ENOSYS);

	srq = pd->device->create_srq(pd, srq_init_attr, NULL);

	if (!IS_ERR(srq)) {
		srq->device    	   = pd->device;
		srq->pd        	   = pd;
		srq->uobject       = NULL;
		srq->event_handler = srq_init_attr->event_handler;
		srq->srq_context   = srq_init_attr->srq_context;
Sean Hefty's avatar
Sean Hefty committed
307
		srq->srq_type      = srq_init_attr->srq_type;
Sean Hefty's avatar
Sean Hefty committed
308
309
310
311
312
313
		if (srq->srq_type == IB_SRQT_XRC) {
			srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
			srq->ext.xrc.cq   = srq_init_attr->ext.xrc.cq;
			atomic_inc(&srq->ext.xrc.xrcd->usecnt);
			atomic_inc(&srq->ext.xrc.cq->usecnt);
		}
314
315
316
317
318
319
320
321
322
323
324
325
		atomic_inc(&pd->usecnt);
		atomic_set(&srq->usecnt, 0);
	}

	return srq;
}
EXPORT_SYMBOL(ib_create_srq);

int ib_modify_srq(struct ib_srq *srq,
		  struct ib_srq_attr *srq_attr,
		  enum ib_srq_attr_mask srq_attr_mask)
{
326
327
328
	return srq->device->modify_srq ?
		srq->device->modify_srq(srq, srq_attr, srq_attr_mask, NULL) :
		-ENOSYS;
329
330
331
332
333
334
335
336
337
338
339
340
341
342
}
EXPORT_SYMBOL(ib_modify_srq);

int ib_query_srq(struct ib_srq *srq,
		 struct ib_srq_attr *srq_attr)
{
	return srq->device->query_srq ?
		srq->device->query_srq(srq, srq_attr) : -ENOSYS;
}
EXPORT_SYMBOL(ib_query_srq);

int ib_destroy_srq(struct ib_srq *srq)
{
	struct ib_pd *pd;
Sean Hefty's avatar
Sean Hefty committed
343
344
345
	enum ib_srq_type srq_type;
	struct ib_xrcd *uninitialized_var(xrcd);
	struct ib_cq *uninitialized_var(cq);
346
347
348
349
350
351
	int ret;

	if (atomic_read(&srq->usecnt))
		return -EBUSY;

	pd = srq->pd;
Sean Hefty's avatar
Sean Hefty committed
352
353
354
355
356
	srq_type = srq->srq_type;
	if (srq_type == IB_SRQT_XRC) {
		xrcd = srq->ext.xrc.xrcd;
		cq = srq->ext.xrc.cq;
	}
357
358

	ret = srq->device->destroy_srq(srq);
Sean Hefty's avatar
Sean Hefty committed
359
	if (!ret) {
360
		atomic_dec(&pd->usecnt);
Sean Hefty's avatar
Sean Hefty committed
361
362
363
364
365
		if (srq_type == IB_SRQT_XRC) {
			atomic_dec(&xrcd->usecnt);
			atomic_dec(&cq->usecnt);
		}
	}
366
367
368
369
370

	return ret;
}
EXPORT_SYMBOL(ib_destroy_srq);

Linus Torvalds's avatar
Linus Torvalds committed
371
372
/* Queue pairs */

373
374
375
static void __ib_shared_qp_event_handler(struct ib_event *event, void *context)
{
	struct ib_qp *qp = context;
376
	unsigned long flags;
377

378
	spin_lock_irqsave(&qp->device->event_handler_lock, flags);
379
	list_for_each_entry(event->element.qp, &qp->open_list, open_list)
380
381
		if (event->element.qp->event_handler)
			event->element.qp->event_handler(event, event->element.qp->qp_context);
382
	spin_unlock_irqrestore(&qp->device->event_handler_lock, flags);
383
384
}

385
386
387
388
389
390
391
static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp)
{
	mutex_lock(&xrcd->tgt_qp_mutex);
	list_add(&qp->xrcd_list, &xrcd->tgt_qp_list);
	mutex_unlock(&xrcd->tgt_qp_mutex);
}

392
393
394
static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp,
				  void (*event_handler)(struct ib_event *, void *),
				  void *qp_context)
395
{
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
	struct ib_qp *qp;
	unsigned long flags;

	qp = kzalloc(sizeof *qp, GFP_KERNEL);
	if (!qp)
		return ERR_PTR(-ENOMEM);

	qp->real_qp = real_qp;
	atomic_inc(&real_qp->usecnt);
	qp->device = real_qp->device;
	qp->event_handler = event_handler;
	qp->qp_context = qp_context;
	qp->qp_num = real_qp->qp_num;
	qp->qp_type = real_qp->qp_type;

	spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
	list_add(&qp->open_list, &real_qp->open_list);
	spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);

	return qp;
}

struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
			 struct ib_qp_open_attr *qp_open_attr)
{
	struct ib_qp *qp, *real_qp;

	if (qp_open_attr->qp_type != IB_QPT_XRC_TGT)
		return ERR_PTR(-EINVAL);

	qp = ERR_PTR(-EINVAL);
427
	mutex_lock(&xrcd->tgt_qp_mutex);
428
429
430
431
432
433
434
	list_for_each_entry(real_qp, &xrcd->tgt_qp_list, xrcd_list) {
		if (real_qp->qp_num == qp_open_attr->qp_num) {
			qp = __ib_open_qp(real_qp, qp_open_attr->event_handler,
					  qp_open_attr->qp_context);
			break;
		}
	}
435
	mutex_unlock(&xrcd->tgt_qp_mutex);
436
	return qp;
437
}
438
EXPORT_SYMBOL(ib_open_qp);
439

Linus Torvalds's avatar
Linus Torvalds committed
440
441
442
struct ib_qp *ib_create_qp(struct ib_pd *pd,
			   struct ib_qp_init_attr *qp_init_attr)
{
443
	struct ib_qp *qp, *real_qp;
Sean Hefty's avatar
Sean Hefty committed
444
	struct ib_device *device;
Linus Torvalds's avatar
Linus Torvalds committed
445

Sean Hefty's avatar
Sean Hefty committed
446
447
	device = pd ? pd->device : qp_init_attr->xrcd->device;
	qp = device->create_qp(pd, qp_init_attr, NULL);
Linus Torvalds's avatar
Linus Torvalds committed
448
449

	if (!IS_ERR(qp)) {
450
451
452
453
		qp->device     = device;
		qp->real_qp    = qp;
		qp->uobject    = NULL;
		qp->qp_type    = qp_init_attr->qp_type;
Sean Hefty's avatar
Sean Hefty committed
454

455
		atomic_set(&qp->usecnt, 0);
Sean Hefty's avatar
Sean Hefty committed
456
		if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) {
457
458
			qp->event_handler = __ib_shared_qp_event_handler;
			qp->qp_context = qp;
Sean Hefty's avatar
Sean Hefty committed
459
460
461
462
463
			qp->pd = NULL;
			qp->send_cq = qp->recv_cq = NULL;
			qp->srq = NULL;
			qp->xrcd = qp_init_attr->xrcd;
			atomic_inc(&qp_init_attr->xrcd->usecnt);
464
465
466
467
468
469
470
471
472
			INIT_LIST_HEAD(&qp->open_list);

			real_qp = qp;
			qp = __ib_open_qp(real_qp, qp_init_attr->event_handler,
					  qp_init_attr->qp_context);
			if (!IS_ERR(qp))
				__ib_insert_xrcd_qp(qp_init_attr->xrcd, real_qp);
			else
				real_qp->device->destroy_qp(real_qp);
Sean Hefty's avatar
Sean Hefty committed
473
		} else {
474
475
			qp->event_handler = qp_init_attr->event_handler;
			qp->qp_context = qp_init_attr->qp_context;
Sean Hefty's avatar
Sean Hefty committed
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
			if (qp_init_attr->qp_type == IB_QPT_XRC_INI) {
				qp->recv_cq = NULL;
				qp->srq = NULL;
			} else {
				qp->recv_cq = qp_init_attr->recv_cq;
				atomic_inc(&qp_init_attr->recv_cq->usecnt);
				qp->srq = qp_init_attr->srq;
				if (qp->srq)
					atomic_inc(&qp_init_attr->srq->usecnt);
			}

			qp->pd	    = pd;
			qp->send_cq = qp_init_attr->send_cq;
			qp->xrcd    = NULL;

			atomic_inc(&pd->usecnt);
			atomic_inc(&qp_init_attr->send_cq->usecnt);
		}
Linus Torvalds's avatar
Linus Torvalds committed
494
495
496
497
498
499
	}

	return qp;
}
EXPORT_SYMBOL(ib_create_qp);

500
501
static const struct {
	int			valid;
Sean Hefty's avatar
Sean Hefty committed
502
	enum ib_qp_attr_mask	req_param[IB_QPT_MAX];
503
	enum ib_qp_attr_mask	req_param_add_eth[IB_QPT_MAX];
Sean Hefty's avatar
Sean Hefty committed
504
	enum ib_qp_attr_mask	opt_param[IB_QPT_MAX];
505
	enum ib_qp_attr_mask	opt_param_add_eth[IB_QPT_MAX];
506
507
508
509
510
511
512
513
514
} qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
	[IB_QPS_RESET] = {
		[IB_QPS_RESET] = { .valid = 1 },
		[IB_QPS_INIT]  = {
			.valid = 1,
			.req_param = {
				[IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
						IB_QP_PORT			|
						IB_QP_QKEY),
Or Gerlitz's avatar
Or Gerlitz committed
515
				[IB_QPT_RAW_PACKET] = IB_QP_PORT,
516
517
518
519
520
521
				[IB_QPT_UC]  = (IB_QP_PKEY_INDEX		|
						IB_QP_PORT			|
						IB_QP_ACCESS_FLAGS),
				[IB_QPT_RC]  = (IB_QP_PKEY_INDEX		|
						IB_QP_PORT			|
						IB_QP_ACCESS_FLAGS),
Sean Hefty's avatar
Sean Hefty committed
522
523
524
525
526
527
				[IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX		|
						IB_QP_PORT			|
						IB_QP_ACCESS_FLAGS),
				[IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX		|
						IB_QP_PORT			|
						IB_QP_ACCESS_FLAGS),
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX		|
						IB_QP_QKEY),
				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX		|
						IB_QP_QKEY),
			}
		},
	},
	[IB_QPS_INIT]  = {
		[IB_QPS_RESET] = { .valid = 1 },
		[IB_QPS_ERR] =   { .valid = 1 },
		[IB_QPS_INIT]  = {
			.valid = 1,
			.opt_param = {
				[IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
						IB_QP_PORT			|
						IB_QP_QKEY),
				[IB_QPT_UC]  = (IB_QP_PKEY_INDEX		|
						IB_QP_PORT			|
						IB_QP_ACCESS_FLAGS),
				[IB_QPT_RC]  = (IB_QP_PKEY_INDEX		|
						IB_QP_PORT			|
						IB_QP_ACCESS_FLAGS),
Sean Hefty's avatar
Sean Hefty committed
550
551
552
553
554
555
				[IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX		|
						IB_QP_PORT			|
						IB_QP_ACCESS_FLAGS),
				[IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX		|
						IB_QP_PORT			|
						IB_QP_ACCESS_FLAGS),
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX		|
						IB_QP_QKEY),
				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX		|
						IB_QP_QKEY),
			}
		},
		[IB_QPS_RTR]   = {
			.valid = 1,
			.req_param = {
				[IB_QPT_UC]  = (IB_QP_AV			|
						IB_QP_PATH_MTU			|
						IB_QP_DEST_QPN			|
						IB_QP_RQ_PSN),
				[IB_QPT_RC]  = (IB_QP_AV			|
						IB_QP_PATH_MTU			|
						IB_QP_DEST_QPN			|
						IB_QP_RQ_PSN			|
						IB_QP_MAX_DEST_RD_ATOMIC	|
						IB_QP_MIN_RNR_TIMER),
Sean Hefty's avatar
Sean Hefty committed
575
576
577
578
579
580
581
582
583
584
				[IB_QPT_XRC_INI] = (IB_QP_AV			|
						IB_QP_PATH_MTU			|
						IB_QP_DEST_QPN			|
						IB_QP_RQ_PSN),
				[IB_QPT_XRC_TGT] = (IB_QP_AV			|
						IB_QP_PATH_MTU			|
						IB_QP_DEST_QPN			|
						IB_QP_RQ_PSN			|
						IB_QP_MAX_DEST_RD_ATOMIC	|
						IB_QP_MIN_RNR_TIMER),
585
			},
586
587
588
589
590
591
			.req_param_add_eth = {
				[IB_QPT_RC]  = (IB_QP_SMAC),
				[IB_QPT_UC]  = (IB_QP_SMAC),
				[IB_QPT_XRC_INI]  = (IB_QP_SMAC),
				[IB_QPT_XRC_TGT]  = (IB_QP_SMAC)
			},
592
593
594
595
596
597
598
599
600
			.opt_param = {
				 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
						 IB_QP_QKEY),
				 [IB_QPT_UC]  = (IB_QP_ALT_PATH			|
						 IB_QP_ACCESS_FLAGS		|
						 IB_QP_PKEY_INDEX),
				 [IB_QPT_RC]  = (IB_QP_ALT_PATH			|
						 IB_QP_ACCESS_FLAGS		|
						 IB_QP_PKEY_INDEX),
Sean Hefty's avatar
Sean Hefty committed
601
602
603
604
605
606
				 [IB_QPT_XRC_INI] = (IB_QP_ALT_PATH		|
						 IB_QP_ACCESS_FLAGS		|
						 IB_QP_PKEY_INDEX),
				 [IB_QPT_XRC_TGT] = (IB_QP_ALT_PATH		|
						 IB_QP_ACCESS_FLAGS		|
						 IB_QP_PKEY_INDEX),
607
608
609
610
				 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX		|
						 IB_QP_QKEY),
				 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX		|
						 IB_QP_QKEY),
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
			 },
			.opt_param_add_eth = {
				[IB_QPT_RC]  = (IB_QP_ALT_SMAC			|
						IB_QP_VID			|
						IB_QP_ALT_VID),
				[IB_QPT_UC]  = (IB_QP_ALT_SMAC			|
						IB_QP_VID			|
						IB_QP_ALT_VID),
				[IB_QPT_XRC_INI]  = (IB_QP_ALT_SMAC			|
						IB_QP_VID			|
						IB_QP_ALT_VID),
				[IB_QPT_XRC_TGT]  = (IB_QP_ALT_SMAC			|
						IB_QP_VID			|
						IB_QP_ALT_VID)
			}
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
		}
	},
	[IB_QPS_RTR]   = {
		[IB_QPS_RESET] = { .valid = 1 },
		[IB_QPS_ERR] =   { .valid = 1 },
		[IB_QPS_RTS]   = {
			.valid = 1,
			.req_param = {
				[IB_QPT_UD]  = IB_QP_SQ_PSN,
				[IB_QPT_UC]  = IB_QP_SQ_PSN,
				[IB_QPT_RC]  = (IB_QP_TIMEOUT			|
						IB_QP_RETRY_CNT			|
						IB_QP_RNR_RETRY			|
						IB_QP_SQ_PSN			|
						IB_QP_MAX_QP_RD_ATOMIC),
Sean Hefty's avatar
Sean Hefty committed
641
642
643
644
645
646
647
				[IB_QPT_XRC_INI] = (IB_QP_TIMEOUT		|
						IB_QP_RETRY_CNT			|
						IB_QP_RNR_RETRY			|
						IB_QP_SQ_PSN			|
						IB_QP_MAX_QP_RD_ATOMIC),
				[IB_QPT_XRC_TGT] = (IB_QP_TIMEOUT		|
						IB_QP_SQ_PSN),
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
				[IB_QPT_SMI] = IB_QP_SQ_PSN,
				[IB_QPT_GSI] = IB_QP_SQ_PSN,
			},
			.opt_param = {
				 [IB_QPT_UD]  = (IB_QP_CUR_STATE		|
						 IB_QP_QKEY),
				 [IB_QPT_UC]  = (IB_QP_CUR_STATE		|
						 IB_QP_ALT_PATH			|
						 IB_QP_ACCESS_FLAGS		|
						 IB_QP_PATH_MIG_STATE),
				 [IB_QPT_RC]  = (IB_QP_CUR_STATE		|
						 IB_QP_ALT_PATH			|
						 IB_QP_ACCESS_FLAGS		|
						 IB_QP_MIN_RNR_TIMER		|
						 IB_QP_PATH_MIG_STATE),
Sean Hefty's avatar
Sean Hefty committed
663
664
665
666
667
668
669
670
671
				 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE		|
						 IB_QP_ALT_PATH			|
						 IB_QP_ACCESS_FLAGS		|
						 IB_QP_PATH_MIG_STATE),
				 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE		|
						 IB_QP_ALT_PATH			|
						 IB_QP_ACCESS_FLAGS		|
						 IB_QP_MIN_RNR_TIMER		|
						 IB_QP_PATH_MIG_STATE),
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
				 [IB_QPT_SMI] = (IB_QP_CUR_STATE		|
						 IB_QP_QKEY),
				 [IB_QPT_GSI] = (IB_QP_CUR_STATE		|
						 IB_QP_QKEY),
			 }
		}
	},
	[IB_QPS_RTS]   = {
		[IB_QPS_RESET] = { .valid = 1 },
		[IB_QPS_ERR] =   { .valid = 1 },
		[IB_QPS_RTS]   = {
			.valid = 1,
			.opt_param = {
				[IB_QPT_UD]  = (IB_QP_CUR_STATE			|
						IB_QP_QKEY),
687
688
				[IB_QPT_UC]  = (IB_QP_CUR_STATE			|
						IB_QP_ACCESS_FLAGS		|
689
690
						IB_QP_ALT_PATH			|
						IB_QP_PATH_MIG_STATE),
691
692
				[IB_QPT_RC]  = (IB_QP_CUR_STATE			|
						IB_QP_ACCESS_FLAGS		|
693
694
695
						IB_QP_ALT_PATH			|
						IB_QP_PATH_MIG_STATE		|
						IB_QP_MIN_RNR_TIMER),
Sean Hefty's avatar
Sean Hefty committed
696
697
698
699
700
701
702
703
704
				[IB_QPT_XRC_INI] = (IB_QP_CUR_STATE		|
						IB_QP_ACCESS_FLAGS		|
						IB_QP_ALT_PATH			|
						IB_QP_PATH_MIG_STATE),
				[IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE		|
						IB_QP_ACCESS_FLAGS		|
						IB_QP_ALT_PATH			|
						IB_QP_PATH_MIG_STATE		|
						IB_QP_MIN_RNR_TIMER),
705
706
707
708
709
710
711
712
713
714
715
716
				[IB_QPT_SMI] = (IB_QP_CUR_STATE			|
						IB_QP_QKEY),
				[IB_QPT_GSI] = (IB_QP_CUR_STATE			|
						IB_QP_QKEY),
			}
		},
		[IB_QPS_SQD]   = {
			.valid = 1,
			.opt_param = {
				[IB_QPT_UD]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
				[IB_QPT_UC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
				[IB_QPT_RC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
Sean Hefty's avatar
Sean Hefty committed
717
718
				[IB_QPT_XRC_INI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
				[IB_QPT_XRC_TGT] = IB_QP_EN_SQD_ASYNC_NOTIFY, /* ??? */
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
				[IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
				[IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY
			}
		},
	},
	[IB_QPS_SQD]   = {
		[IB_QPS_RESET] = { .valid = 1 },
		[IB_QPS_ERR] =   { .valid = 1 },
		[IB_QPS_RTS]   = {
			.valid = 1,
			.opt_param = {
				[IB_QPT_UD]  = (IB_QP_CUR_STATE			|
						IB_QP_QKEY),
				[IB_QPT_UC]  = (IB_QP_CUR_STATE			|
						IB_QP_ALT_PATH			|
						IB_QP_ACCESS_FLAGS		|
						IB_QP_PATH_MIG_STATE),
				[IB_QPT_RC]  = (IB_QP_CUR_STATE			|
						IB_QP_ALT_PATH			|
						IB_QP_ACCESS_FLAGS		|
						IB_QP_MIN_RNR_TIMER		|
						IB_QP_PATH_MIG_STATE),
Sean Hefty's avatar
Sean Hefty committed
741
742
743
744
745
746
747
748
749
				[IB_QPT_XRC_INI] = (IB_QP_CUR_STATE		|
						IB_QP_ALT_PATH			|
						IB_QP_ACCESS_FLAGS		|
						IB_QP_PATH_MIG_STATE),
				[IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE		|
						IB_QP_ALT_PATH			|
						IB_QP_ACCESS_FLAGS		|
						IB_QP_MIN_RNR_TIMER		|
						IB_QP_PATH_MIG_STATE),
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
				[IB_QPT_SMI] = (IB_QP_CUR_STATE			|
						IB_QP_QKEY),
				[IB_QPT_GSI] = (IB_QP_CUR_STATE			|
						IB_QP_QKEY),
			}
		},
		[IB_QPS_SQD]   = {
			.valid = 1,
			.opt_param = {
				[IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
						IB_QP_QKEY),
				[IB_QPT_UC]  = (IB_QP_AV			|
						IB_QP_ALT_PATH			|
						IB_QP_ACCESS_FLAGS		|
						IB_QP_PKEY_INDEX		|
						IB_QP_PATH_MIG_STATE),
				[IB_QPT_RC]  = (IB_QP_PORT			|
						IB_QP_AV			|
						IB_QP_TIMEOUT			|
						IB_QP_RETRY_CNT			|
						IB_QP_RNR_RETRY			|
						IB_QP_MAX_QP_RD_ATOMIC		|
						IB_QP_MAX_DEST_RD_ATOMIC	|
						IB_QP_ALT_PATH			|
						IB_QP_ACCESS_FLAGS		|
						IB_QP_PKEY_INDEX		|
						IB_QP_MIN_RNR_TIMER		|
						IB_QP_PATH_MIG_STATE),
Sean Hefty's avatar
Sean Hefty committed
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
				[IB_QPT_XRC_INI] = (IB_QP_PORT			|
						IB_QP_AV			|
						IB_QP_TIMEOUT			|
						IB_QP_RETRY_CNT			|
						IB_QP_RNR_RETRY			|
						IB_QP_MAX_QP_RD_ATOMIC		|
						IB_QP_ALT_PATH			|
						IB_QP_ACCESS_FLAGS		|
						IB_QP_PKEY_INDEX		|
						IB_QP_PATH_MIG_STATE),
				[IB_QPT_XRC_TGT] = (IB_QP_PORT			|
						IB_QP_AV			|
						IB_QP_TIMEOUT			|
						IB_QP_MAX_DEST_RD_ATOMIC	|
						IB_QP_ALT_PATH			|
						IB_QP_ACCESS_FLAGS		|
						IB_QP_PKEY_INDEX		|
						IB_QP_MIN_RNR_TIMER		|
						IB_QP_PATH_MIG_STATE),
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX		|
						IB_QP_QKEY),
				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX		|
						IB_QP_QKEY),
			}
		}
	},
	[IB_QPS_SQE]   = {
		[IB_QPS_RESET] = { .valid = 1 },
		[IB_QPS_ERR] =   { .valid = 1 },
		[IB_QPS_RTS]   = {
			.valid = 1,
			.opt_param = {
				[IB_QPT_UD]  = (IB_QP_CUR_STATE			|
						IB_QP_QKEY),
				[IB_QPT_UC]  = (IB_QP_CUR_STATE			|
						IB_QP_ACCESS_FLAGS),
				[IB_QPT_SMI] = (IB_QP_CUR_STATE			|
						IB_QP_QKEY),
				[IB_QPT_GSI] = (IB_QP_CUR_STATE			|
						IB_QP_QKEY),
			}
		}
	},
	[IB_QPS_ERR] = {
		[IB_QPS_RESET] = { .valid = 1 },
		[IB_QPS_ERR] =   { .valid = 1 }
	}
};

int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
828
829
		       enum ib_qp_type type, enum ib_qp_attr_mask mask,
		       enum rdma_link_layer ll)
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
{
	enum ib_qp_attr_mask req_param, opt_param;

	if (cur_state  < 0 || cur_state  > IB_QPS_ERR ||
	    next_state < 0 || next_state > IB_QPS_ERR)
		return 0;

	if (mask & IB_QP_CUR_STATE  &&
	    cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS &&
	    cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE)
		return 0;

	if (!qp_state_table[cur_state][next_state].valid)
		return 0;

	req_param = qp_state_table[cur_state][next_state].req_param[type];
	opt_param = qp_state_table[cur_state][next_state].opt_param[type];

848
849
850
851
852
853
854
	if (ll == IB_LINK_LAYER_ETHERNET) {
		req_param |= qp_state_table[cur_state][next_state].
			req_param_add_eth[type];
		opt_param |= qp_state_table[cur_state][next_state].
			opt_param_add_eth[type];
	}

855
856
857
858
859
860
861
862
863
864
	if ((mask & req_param) != req_param)
		return 0;

	if (mask & ~(req_param | opt_param | IB_QP_STATE))
		return 0;

	return 1;
}
EXPORT_SYMBOL(ib_modify_qp_is_ok);

865
866
867
868
869
870
871
int ib_resolve_eth_l2_attrs(struct ib_qp *qp,
			    struct ib_qp_attr *qp_attr, int *qp_attr_mask)
{
	int           ret = 0;
	union ib_gid  sgid;

	if ((*qp_attr_mask & IB_QP_AV)  &&
872
	    (rdma_cap_eth_ah(qp->device, qp_attr->ah_attr.port_num))) {
873
874
875
876
877
878
879
		ret = ib_query_gid(qp->device, qp_attr->ah_attr.port_num,
				   qp_attr->ah_attr.grh.sgid_index, &sgid);
		if (ret)
			goto out;
		if (rdma_link_local_addr((struct in6_addr *)qp_attr->ah_attr.grh.dgid.raw)) {
			rdma_get_ll_mac((struct in6_addr *)qp_attr->ah_attr.grh.dgid.raw, qp_attr->ah_attr.dmac);
			rdma_get_ll_mac((struct in6_addr *)sgid.raw, qp_attr->smac);
880
881
			if (!(*qp_attr_mask & IB_QP_VID))
				qp_attr->vlan_id = rdma_get_vlan_id(&sgid);
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
		} else {
			ret = rdma_addr_find_dmac_by_grh(&sgid, &qp_attr->ah_attr.grh.dgid,
					qp_attr->ah_attr.dmac, &qp_attr->vlan_id);
			if (ret)
				goto out;
			ret = rdma_addr_find_smac_by_sgid(&sgid, qp_attr->smac, NULL);
			if (ret)
				goto out;
		}
		*qp_attr_mask |= IB_QP_SMAC;
		if (qp_attr->vlan_id < 0xFFFF)
			*qp_attr_mask |= IB_QP_VID;
	}
out:
	return ret;
}
EXPORT_SYMBOL(ib_resolve_eth_l2_attrs);


Linus Torvalds's avatar
Linus Torvalds committed
901
902
903
904
int ib_modify_qp(struct ib_qp *qp,
		 struct ib_qp_attr *qp_attr,
		 int qp_attr_mask)
{
905
906
907
908
909
910
	int ret;

	ret = ib_resolve_eth_l2_attrs(qp, qp_attr, &qp_attr_mask);
	if (ret)
		return ret;

911
	return qp->device->modify_qp(qp->real_qp, qp_attr, qp_attr_mask, NULL);
Linus Torvalds's avatar
Linus Torvalds committed
912
913
914
915
916
917
918
919
920
}
EXPORT_SYMBOL(ib_modify_qp);

int ib_query_qp(struct ib_qp *qp,
		struct ib_qp_attr *qp_attr,
		int qp_attr_mask,
		struct ib_qp_init_attr *qp_init_attr)
{
	return qp->device->query_qp ?
921
		qp->device->query_qp(qp->real_qp, qp_attr, qp_attr_mask, qp_init_attr) :
Linus Torvalds's avatar
Linus Torvalds committed
922
923
924
925
		-ENOSYS;
}
EXPORT_SYMBOL(ib_query_qp);

926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
int ib_close_qp(struct ib_qp *qp)
{
	struct ib_qp *real_qp;
	unsigned long flags;

	real_qp = qp->real_qp;
	if (real_qp == qp)
		return -EINVAL;

	spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
	list_del(&qp->open_list);
	spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);

	atomic_dec(&real_qp->usecnt);
	kfree(qp);

	return 0;
}
EXPORT_SYMBOL(ib_close_qp);

static int __ib_destroy_shared_qp(struct ib_qp *qp)
{
	struct ib_xrcd *xrcd;
	struct ib_qp *real_qp;
	int ret;

	real_qp = qp->real_qp;
	xrcd = real_qp->xrcd;

	mutex_lock(&xrcd->tgt_qp_mutex);
	ib_close_qp(qp);
	if (atomic_read(&real_qp->usecnt) == 0)
		list_del(&real_qp->xrcd_list);
	else
		real_qp = NULL;
	mutex_unlock(&xrcd->tgt_qp_mutex);

	if (real_qp) {
		ret = ib_destroy_qp(real_qp);
		if (!ret)
			atomic_dec(&xrcd->usecnt);
		else
			__ib_insert_xrcd_qp(xrcd, real_qp);
	}

	return 0;
}

Linus Torvalds's avatar
Linus Torvalds committed
974
975
976
977
978
979
980
int ib_destroy_qp(struct ib_qp *qp)
{
	struct ib_pd *pd;
	struct ib_cq *scq, *rcq;
	struct ib_srq *srq;
	int ret;

981
982
983
984
985
986
	if (atomic_read(&qp->usecnt))
		return -EBUSY;

	if (qp->real_qp != qp)
		return __ib_destroy_shared_qp(qp);

Sean Hefty's avatar
Sean Hefty committed
987
988
989
990
	pd   = qp->pd;
	scq  = qp->send_cq;
	rcq  = qp->recv_cq;
	srq  = qp->srq;
Linus Torvalds's avatar
Linus Torvalds committed
991
992
993

	ret = qp->device->destroy_qp(qp);
	if (!ret) {
Sean Hefty's avatar
Sean Hefty committed
994
995
996
997
998
999
		if (pd)
			atomic_dec(&pd->usecnt);
		if (scq)
			atomic_dec(&scq->usecnt);
		if (rcq)
			atomic_dec(&rcq->usecnt);
Linus Torvalds's avatar
Linus Torvalds committed
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
		if (srq)
			atomic_dec(&srq->usecnt);
	}

	return ret;
}
EXPORT_SYMBOL(ib_destroy_qp);

/* Completion queues */

struct ib_cq *ib_create_cq(struct ib_device *device,
			   ib_comp_handler comp_handler,
			   void (*event_handler)(struct ib_event *, void *),
1013
			   void *cq_context, int cqe, int comp_vector)
Linus Torvalds's avatar
Linus Torvalds committed
1014
1015
1016
{
	struct ib_cq *cq;

1017
	cq = device->create_cq(device, cqe, comp_vector, NULL, NULL);
Linus Torvalds's avatar
Linus Torvalds committed
1018
1019
1020

	if (!IS_ERR(cq)) {
		cq->device        = device;
1021
		cq->uobject       = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
		cq->comp_handler  = comp_handler;
		cq->event_handler = event_handler;
		cq->cq_context    = cq_context;
		atomic_set(&cq->usecnt, 0);
	}

	return cq;
}
EXPORT_SYMBOL(ib_create_cq);

Eli Cohen's avatar
Eli Cohen committed
1032
1033
1034
1035
1036
1037
1038
int ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
{
	return cq->device->modify_cq ?
		cq->device->modify_cq(cq, cq_count, cq_period) : -ENOSYS;
}
EXPORT_SYMBOL(ib_modify_cq);

Linus Torvalds's avatar
Linus Torvalds committed
1039
1040
1041
1042
1043
1044
1045
1046
1047
int ib_destroy_cq(struct ib_cq *cq)
{
	if (atomic_read(&cq->usecnt))
		return -EBUSY;

	return cq->device->destroy_cq(cq);
}
EXPORT_SYMBOL(ib_destroy_cq);

Roland Dreier's avatar
Roland Dreier committed
1048
int ib_resize_cq(struct ib_cq *cq, int cqe)
Linus Torvalds's avatar
Linus Torvalds committed
1049
{
1050
	return cq->device->resize_cq ?
1051
		cq->device->resize_cq(cq, cqe, NULL) : -ENOSYS;
Linus Torvalds's avatar
Linus Torvalds committed
1052
1053
1054
1055
1056
1057
1058
1059
}
EXPORT_SYMBOL(ib_resize_cq);

/* Memory regions */

struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
{
	struct ib_mr *mr;
1060
1061
1062
1063
1064
	int err;

	err = ib_check_mr_access(mr_access_flags);
	if (err)
		return ERR_PTR(err);
Linus Torvalds's avatar
Linus Torvalds committed
1065
1066
1067
1068

	mr = pd->device->get_dma_mr(pd, mr_access_flags);

	if (!IS_ERR(mr)) {
1069
1070
1071
		mr->device  = pd->device;
		mr->pd      = pd;
		mr->uobject = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
		atomic_inc(&pd->usecnt);
		atomic_set(&mr->usecnt, 0);
	}

	return mr;
}
EXPORT_SYMBOL(ib_get_dma_mr);

struct ib_mr *ib_reg_phys_mr(struct ib_pd *pd,
			     struct ib_phys_buf *phys_buf_array,
			     int num_phys_buf,
			     int mr_access_flags,
			     u64 *iova_start)
{
	struct ib_mr *mr;
1087
1088
1089
1090
1091
	int err;

	err = ib_check_mr_access(mr_access_flags);
	if (err)
		return ERR_PTR(err);
Linus Torvalds's avatar
Linus Torvalds committed
1092

1093
1094
1095
	if (!pd->device->reg_phys_mr)
		return ERR_PTR(-ENOSYS);

Linus Torvalds's avatar
Linus Torvalds committed
1096
1097
1098
1099
	mr = pd->device->reg_phys_mr(pd, phys_buf_array, num_phys_buf,
				     mr_access_flags, iova_start);

	if (!IS_ERR(mr)) {
1100
1101
1102
		mr->device  = pd->device;
		mr->pd      = pd;
		mr->uobject = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
		atomic_inc(&pd->usecnt);
		atomic_set(&mr->usecnt, 0);
	}

	return mr;
}
EXPORT_SYMBOL(ib_reg_phys_mr);

int ib_rereg_phys_mr(struct ib_mr *mr,
		     int mr_rereg_mask,
		     struct ib_pd *pd,
		     struct ib_phys_buf *phys_buf_array,
		     int num_phys_buf,
		     int mr_access_flags,
		     u64 *iova_start)
{
	struct ib_pd *old_pd;
	int ret;

1122
1123
1124
1125
	ret = ib_check_mr_access(mr_access_flags);
	if (ret)
		return ret;

Linus Torvalds's avatar
Linus Torvalds committed
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
	if (!mr->device->rereg_phys_mr)
		return -ENOSYS;

	if (atomic_read(&mr->usecnt))
		return -EBUSY;

	old_pd = mr->pd;

	ret = mr->device->rereg_phys_mr(mr, mr_rereg_mask, pd,
					phys_buf_array, num_phys_buf,
					mr_access_flags, iova_start);

	if (!ret && (mr_rereg_mask & IB_MR_REREG_PD)) {
		atomic_dec(&old_pd->usecnt);
		atomic_inc(&pd->usecnt);
	}

	return ret;
}
EXPORT_SYMBOL(ib_rereg_phys_mr);

int ib_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
{
	return mr->device->query_mr ?
		mr->device->query_mr(mr, mr_attr) : -ENOSYS;
}
EXPORT_SYMBOL(ib_query_mr);

int ib_dereg_mr(struct ib_mr *mr)
{
	struct ib_pd *pd;
	int ret;

	if (atomic_read(&mr->usecnt))
		return -EBUSY;

	pd = mr->pd;
	ret = mr->device->dereg_mr(mr);
	if (!ret)
		atomic_dec(&pd->usecnt);

	return ret;
}
EXPORT_SYMBOL(ib_dereg_mr);

1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
struct ib_mr *ib_create_mr(struct ib_pd *pd,
			   struct ib_mr_init_attr *mr_init_attr)
{
	struct ib_mr *mr;

	if (!pd->device->create_mr)
		return ERR_PTR(-ENOSYS);

	mr = pd->device->create_mr(pd, mr_init_attr);

	if (!IS_ERR(mr)) {
		mr->device  = pd->device;
		mr->pd      = pd;
		mr->uobject = NULL;
		atomic_inc(&pd->usecnt);
		atomic_set(&mr->usecnt, 0);
	}

	return mr;
}
EXPORT_SYMBOL(ib_create_mr);

int ib_destroy_mr(struct ib_mr *mr)
{
	struct ib_pd *pd;
	int ret;

	if (atomic_read(&mr->usecnt))
		return -EBUSY;

	pd = mr->pd;
	ret = mr->device->destroy_mr(mr);
	if (!ret)
		atomic_dec(&pd->usecnt);

	return ret;
}
EXPORT_SYMBOL(ib_destroy_mr);

1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
struct ib_mr *ib_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len)
{
	struct ib_mr *mr;

	if (!pd->device->alloc_fast_reg_mr)
		return ERR_PTR(-ENOSYS);

	mr = pd->device->alloc_fast_reg_mr(pd, max_page_list_len);

	if (!IS_ERR(mr)) {
		mr->device  = pd->device;
		mr->pd      = pd;
		mr->uobject = NULL;
		atomic_inc(&pd->usecnt);
		atomic_set(&mr->usecnt, 0);
	}

	return mr;
}
EXPORT_SYMBOL(ib_alloc_fast_reg_mr);

struct ib_fast_reg_page_list *ib_alloc_fast_reg_page_list(struct ib_device *device,
							  int max_page_list_len)
{
	struct ib_fast_reg_page_list *page_list;

	if (!device->alloc_fast_reg_page_list)
		return ERR_PTR(-ENOSYS);

	page_list = device->alloc_fast_reg_page_list(device, max_page_list_len);

	if (!IS_ERR(page_list)) {
		page_list->device = device;
		page_list->max_page_list_len = max_page_list_len;
	}

	return page_list;
}
EXPORT_SYMBOL(ib_alloc_fast_reg_page_list);

void ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list)
{
	page_list->device->free_fast_reg_page_list(page_list);
}
EXPORT_SYMBOL(ib_free_fast_reg_page_list);

Linus Torvalds's avatar
Linus Torvalds committed
1256
1257
/* Memory windows */

1258
struct ib_mw *ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type)
Linus Torvalds's avatar
Linus Torvalds committed
1259
1260
1261
1262
1263
1264
{
	struct ib_mw *mw;

	if (!pd->device->alloc_mw)
		return ERR_PTR(-ENOSYS);

1265
	mw = pd->device->alloc_mw(pd, type);
Linus Torvalds's avatar
Linus Torvalds committed
1266
	if (!IS_ERR(mw)) {
1267
1268
1269
		mw->device  = pd->device;
		mw->pd      = pd;
		mw->uobject = NULL;
1270
		mw->type    = type;
Linus Torvalds's avatar
Linus Torvalds committed
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
		atomic_inc(&pd->usecnt);
	}

	return mw;
}
EXPORT_SYMBOL(ib_alloc_mw);

int ib_dealloc_mw(struct ib_mw *mw)
{
	struct ib_pd *pd;
	int ret;

	pd = mw->pd;
	ret = mw->device->dealloc_mw(mw);
	if (!ret)
		atomic_dec(&pd->usecnt);

	return ret;
}
EXPORT_SYMBOL(ib_dealloc_mw);

/* "Fast" memory regions */

struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd,
			    int mr_access_flags,
			    struct ib_fmr_attr *fmr_attr)
{
	struct ib_fmr *fmr;

	if (!pd->device->alloc_fmr)
		return ERR_PTR(-ENOSYS);

	fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr);
	if (!IS_ERR(fmr)) {
		fmr->device = pd->device;
		fmr->pd     = pd;
		atomic_inc(&pd->usecnt);
	}

	return fmr;
}
EXPORT_SYMBOL(ib_alloc_fmr);

int ib_unmap_fmr(struct list_head *fmr_list)
{
	struct ib_fmr *fmr;

	if (list_empty(fmr_list))
		return 0;

	fmr = list_entry(fmr_list->next, struct ib_fmr, list);
	return fmr->device->unmap_fmr(fmr_list);
}
EXPORT_SYMBOL(ib_unmap_fmr);

int ib_dealloc_fmr(struct ib_fmr *fmr)
{
	struct ib_pd *pd;
	int ret;

	pd = fmr->pd;
	ret = fmr->device->dealloc_fmr(fmr);
	if (!ret)
		atomic_dec(&pd->usecnt);

	return ret;
}
EXPORT_SYMBOL(ib_dealloc_fmr);

/* Multicast groups */

int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
{
1344
1345
	int ret;

1346
1347
1348
1349
1350
	if (!qp->device->attach_mcast)
		return -ENOSYS;
	if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
		return -EINVAL;

1351
1352
1353
1354
	ret = qp->device->attach_mcast(qp, gid, lid);
	if (!ret)
		atomic_inc(&qp->usecnt);
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
1355
1356
1357
1358
1359
}
EXPORT_SYMBOL(ib_attach_mcast);

int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
{
1360
1361
	int ret;

1362
1363
1364
1365
1366
	if (!qp->device->detach_mcast)
		return -ENOSYS;
	if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
		return -EINVAL;

1367
1368
1369
1370
	ret = qp->device->detach_mcast(qp, gid, lid);
	if (!ret)
		atomic_dec(&qp->usecnt);
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
1371
1372
}
EXPORT_SYMBOL(ib_detach_mcast);
Sean Hefty's avatar
Sean Hefty committed
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383

struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device)
{
	struct ib_xrcd *xrcd;

	if (!device->alloc_xrcd)
		return ERR_PTR(-ENOSYS);

	xrcd = device->alloc_xrcd(device, NULL, NULL);
	if (!IS_ERR(xrcd)) {
		xrcd->device = device;
1384
		xrcd->inode = NULL;
Sean Hefty's avatar
Sean Hefty committed
1385
		atomic_set(&xrcd->usecnt, 0);
1386
1387
		mutex_init(&xrcd->tgt_qp_mutex);
		INIT_LIST_HEAD(&xrcd->tgt_qp_list);
Sean Hefty's avatar
Sean Hefty committed
1388
1389
1390
1391
1392
1393
1394
1395
	}

	return xrcd;
}
EXPORT_SYMBOL(ib_alloc_xrcd);

int ib_dealloc_xrcd(struct ib_xrcd *xrcd)
{
1396
1397
1398
	struct ib_qp *qp;
	int ret;

Sean Hefty's avatar
Sean Hefty committed
1399
1400
1401
	if (atomic_read(&xrcd->usecnt))
		return -EBUSY;

1402
1403
1404
1405
1406
1407
1408
	while (!list_empty(&xrcd->tgt_qp_list)) {
		qp = list_entry(xrcd->tgt_qp_list.next, struct ib_qp, xrcd_list);
		ret = ib_destroy_qp(qp);
		if (ret)
			return ret;
	}

Sean Hefty's avatar
Sean Hefty committed
1409
1410
1411
	return xrcd->device->dealloc_xrcd(xrcd);
}
EXPORT_SYMBOL(ib_dealloc_xrcd);
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438

struct ib_flow *ib_create_flow(struct ib_qp *qp,
			       struct ib_flow_attr *flow_attr,
			       int domain)
{
	struct ib_flow *flow_id;
	if (!qp->device->create_flow)
		return ERR_PTR(-ENOSYS);

	flow_id = qp->device->create_flow(qp, flow_attr, domain);
	if (!IS_ERR(flow_id))
		atomic_inc(&qp->usecnt);
	return flow_id;
}
EXPORT_SYMBOL(ib_create_flow);

int ib_destroy_flow(struct ib_flow *flow_id)
{
	int err;
	struct ib_qp *qp = flow_id->qp;

	err = qp->device->destroy_flow(flow_id);
	if (!err)
		atomic_dec(&qp->usecnt);
	return err;
}
EXPORT_SYMBOL(ib_destroy_flow);
1439
1440
1441
1442
1443
1444
1445
1446

int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,
		       struct ib_mr_status *mr_status)
{
	return mr->device->check_mr_status ?
		mr->device->check_mr_status(mr, check_mask, mr_status) : -ENOSYS;
}
EXPORT_SYMBOL(ib_check_mr_status);