ixgbevf_main.c 114 KB
Newer Older
1
2
3
/*******************************************************************************

  Intel 82599 Virtual Function driver
4
  Copyright(c) 1999 - 2015 Intel Corporation.
5
6
7
8
9
10
11
12
13
14
15

  This program is free software; you can redistribute it and/or modify it
  under the terms and conditions of the GNU General Public License,
  version 2, as published by the Free Software Foundation.

  This program is distributed in the hope it will be useful, but WITHOUT
  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  more details.

  You should have received a copy of the GNU General Public License along with
16
  this program; if not, see <http://www.gnu.org/licenses/>.
17
18
19
20
21
22
23
24
25
26
27
28
29

  The full GNU General Public License is included in this distribution in
  the file called "COPYING".

  Contact Information:
  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497

*******************************************************************************/

/******************************************************************************
 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
******************************************************************************/
30
31
32

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

33
#include <linux/types.h>
Jiri Pirko's avatar
Jiri Pirko committed
34
#include <linux/bitops.h>
35
36
37
38
39
40
41
42
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/netdevice.h>
#include <linux/vmalloc.h>
#include <linux/string.h>
#include <linux/in.h>
#include <linux/ip.h>
#include <linux/tcp.h>
43
#include <linux/sctp.h>
44
#include <linux/ipv6.h>
45
#include <linux/slab.h>
46
47
48
#include <net/checksum.h>
#include <net/ip6_checksum.h>
#include <linux/ethtool.h>
49
#include <linux/if.h>
50
#include <linux/if_vlan.h>
51
#include <linux/prefetch.h>
52
53
54

#include "ixgbevf.h"

55
const char ixgbevf_driver_name[] = "ixgbevf";
56
static const char ixgbevf_driver_string[] =
Greg Rose's avatar
Greg Rose committed
57
	"Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
58

Don Skidmore's avatar
Don Skidmore committed
59
#define DRV_VERSION "2.12.1-k"
60
const char ixgbevf_driver_version[] = DRV_VERSION;
61
static char ixgbevf_copyright[] =
Greg Rose's avatar
Greg Rose committed
62
	"Copyright (c) 2009 - 2012 Intel Corporation.";
63
64

static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
65
66
	[board_82599_vf] = &ixgbevf_82599_vf_info,
	[board_X540_vf]  = &ixgbevf_X540_vf_info,
67
68
	[board_X550_vf]  = &ixgbevf_X550_vf_info,
	[board_X550EM_x_vf] = &ixgbevf_X550EM_x_vf_info,
69
70
71
72
73
74
75
76
77
78
};

/* ixgbevf_pci_tbl - PCI Device ID Table
 *
 * Wildcard entries (PCI_ANY_ID) should come last
 * Last entry must be all 0s
 *
 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
 *   Class, Class Mask, private data (not used) }
 */
79
static const struct pci_device_id ixgbevf_pci_tbl[] = {
80
81
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf },
82
83
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF), board_X550_vf },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF), board_X550EM_x_vf },
84
85
86
87
88
89
	/* required last entry */
	{0, }
};
MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);

MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
90
MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver");
91
92
93
MODULE_LICENSE("GPL");
MODULE_VERSION(DRV_VERSION);

94
95
96
97
#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
static int debug = -1;
module_param(debug, int, 0);
MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
98

99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
static void ixgbevf_service_event_schedule(struct ixgbevf_adapter *adapter)
{
	if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
	    !test_bit(__IXGBEVF_REMOVING, &adapter->state) &&
	    !test_and_set_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state))
		schedule_work(&adapter->service_task);
}

static void ixgbevf_service_event_complete(struct ixgbevf_adapter *adapter)
{
	BUG_ON(!test_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state));

	/* flush memory to make sure state is correct before next watchdog */
	smp_mb__before_atomic();
	clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state);
}

116
/* forward decls */
117
static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter);
118
static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);
119
static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter);
120

121
122
123
124
125
126
127
128
static void ixgbevf_remove_adapter(struct ixgbe_hw *hw)
{
	struct ixgbevf_adapter *adapter = hw->back;

	if (!hw->hw_addr)
		return;
	hw->hw_addr = NULL;
	dev_err(&adapter->pdev->dev, "Adapter removed\n");
129
130
	if (test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state))
		ixgbevf_service_event_schedule(adapter);
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
}

static void ixgbevf_check_remove(struct ixgbe_hw *hw, u32 reg)
{
	u32 value;

	/* The following check not only optimizes a bit by not
	 * performing a read on the status register when the
	 * register just read was a status register read that
	 * returned IXGBE_FAILED_READ_REG. It also blocks any
	 * potential recursion.
	 */
	if (reg == IXGBE_VFSTATUS) {
		ixgbevf_remove_adapter(hw);
		return;
	}
147
	value = ixgbevf_read_reg(hw, IXGBE_VFSTATUS);
148
149
150
151
	if (value == IXGBE_FAILED_READ_REG)
		ixgbevf_remove_adapter(hw);
}

152
u32 ixgbevf_read_reg(struct ixgbe_hw *hw, u32 reg)
153
154
155
156
157
158
159
160
161
162
163
164
{
	u8 __iomem *reg_addr = ACCESS_ONCE(hw->hw_addr);
	u32 value;

	if (IXGBE_REMOVED(reg_addr))
		return IXGBE_FAILED_READ_REG;
	value = readl(reg_addr + reg);
	if (unlikely(value == IXGBE_FAILED_READ_REG))
		ixgbevf_check_remove(hw, reg);
	return value;
}

165
/**
166
 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
167
168
169
170
 * @adapter: pointer to adapter struct
 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
 * @queue: queue to map the corresponding interrupt to
 * @msix_vector: the vector to map to the corresponding queue
171
 **/
172
173
174
175
176
static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
			     u8 queue, u8 msix_vector)
{
	u32 ivar, index;
	struct ixgbe_hw *hw = &adapter->hw;
177

178
179
180
181
182
183
184
185
	if (direction == -1) {
		/* other causes */
		msix_vector |= IXGBE_IVAR_ALLOC_VAL;
		ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
		ivar &= ~0xFF;
		ivar |= msix_vector;
		IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
	} else {
186
		/* Tx or Rx causes */
187
188
189
190
191
192
193
194
195
		msix_vector |= IXGBE_IVAR_ALLOC_VAL;
		index = ((16 * (queue & 1)) + (8 * direction));
		ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
		ivar &= ~(0xFF << index);
		ivar |= (msix_vector << index);
		IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
	}
}

196
static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring,
197
198
199
200
201
					struct ixgbevf_tx_buffer *tx_buffer)
{
	if (tx_buffer->skb) {
		dev_kfree_skb_any(tx_buffer->skb);
		if (dma_unmap_len(tx_buffer, len))
202
			dma_unmap_single(tx_ring->dev,
203
204
					 dma_unmap_addr(tx_buffer, dma),
					 dma_unmap_len(tx_buffer, len),
205
					 DMA_TO_DEVICE);
206
207
208
209
210
	} else if (dma_unmap_len(tx_buffer, len)) {
		dma_unmap_page(tx_ring->dev,
			       dma_unmap_addr(tx_buffer, dma),
			       dma_unmap_len(tx_buffer, len),
			       DMA_TO_DEVICE);
211
	}
212
213
214
215
	tx_buffer->next_to_watch = NULL;
	tx_buffer->skb = NULL;
	dma_unmap_len_set(tx_buffer, len, 0);
	/* tx_buffer must be completely set up in the transmit path */
216
217
}

218
219
220
221
static u64 ixgbevf_get_tx_completed(struct ixgbevf_ring *ring)
{
	return ring->stats.packets;
}
222

223
224
225
226
static u32 ixgbevf_get_tx_pending(struct ixgbevf_ring *ring)
{
	struct ixgbevf_adapter *adapter = netdev_priv(ring->netdev);
	struct ixgbe_hw *hw = &adapter->hw;
227

228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
	u32 head = IXGBE_READ_REG(hw, IXGBE_VFTDH(ring->reg_idx));
	u32 tail = IXGBE_READ_REG(hw, IXGBE_VFTDT(ring->reg_idx));

	if (head != tail)
		return (head < tail) ?
			tail - head : (tail + ring->count - head);

	return 0;
}

static inline bool ixgbevf_check_tx_hang(struct ixgbevf_ring *tx_ring)
{
	u32 tx_done = ixgbevf_get_tx_completed(tx_ring);
	u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
	u32 tx_pending = ixgbevf_get_tx_pending(tx_ring);

	clear_check_for_tx_hang(tx_ring);

	/* Check for a hung queue, but be thorough. This verifies
	 * that a transmit has been completed since the previous
	 * check AND there is at least one packet pending. The
	 * ARMED bit is set to indicate a potential hang.
	 */
	if ((tx_done_old == tx_done) && tx_pending) {
		/* make sure it is true for two checks in a row */
		return test_and_set_bit(__IXGBEVF_HANG_CHECK_ARMED,
					&tx_ring->state);
	}
	/* reset the countdown */
	clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &tx_ring->state);

	/* update completed stats and continue */
	tx_ring->tx_stats.tx_done_old = tx_done;

	return false;
}

265
266
267
268
269
270
271
272
273
static void ixgbevf_tx_timeout_reset(struct ixgbevf_adapter *adapter)
{
	/* Do the reset outside of interrupt context */
	if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) {
		adapter->flags |= IXGBEVF_FLAG_RESET_REQUESTED;
		ixgbevf_service_event_schedule(adapter);
	}
}

274
275
276
277
278
279
280
281
/**
 * ixgbevf_tx_timeout - Respond to a Tx Hang
 * @netdev: network interface device structure
 **/
static void ixgbevf_tx_timeout(struct net_device *netdev)
{
	struct ixgbevf_adapter *adapter = netdev_priv(netdev);

282
	ixgbevf_tx_timeout_reset(adapter);
283
}
284
285
286

/**
 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
287
 * @q_vector: board private structure
288
289
 * @tx_ring: tx ring to clean
 **/
290
static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
291
292
				 struct ixgbevf_ring *tx_ring)
{
293
	struct ixgbevf_adapter *adapter = q_vector->adapter;
294
295
	struct ixgbevf_tx_buffer *tx_buffer;
	union ixgbe_adv_tx_desc *tx_desc;
296
	unsigned int total_bytes = 0, total_packets = 0;
297
298
	unsigned int budget = tx_ring->count / 2;
	unsigned int i = tx_ring->next_to_clean;
299

300
301
302
	if (test_bit(__IXGBEVF_DOWN, &adapter->state))
		return true;

303
304
305
	tx_buffer = &tx_ring->tx_buffer_info[i];
	tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
	i -= tx_ring->count;
306

307
	do {
308
		union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
309
310
311
312
313
314
315
316
317
318
319
320
321

		/* if next_to_watch is not set then there is no work pending */
		if (!eop_desc)
			break;

		/* prevent any other reads prior to eop_desc */
		read_barrier_depends();

		/* if DD is not set pending work has not been completed */
		if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
			break;

		/* clear next_to_watch to prevent false hangs */
322
		tx_buffer->next_to_watch = NULL;
323

324
325
326
		/* update the statistics for this packet */
		total_bytes += tx_buffer->bytecount;
		total_packets += tx_buffer->gso_segs;
327

328
329
330
331
332
333
334
335
336
		/* free the skb */
		dev_kfree_skb_any(tx_buffer->skb);

		/* unmap skb header data */
		dma_unmap_single(tx_ring->dev,
				 dma_unmap_addr(tx_buffer, dma),
				 dma_unmap_len(tx_buffer, len),
				 DMA_TO_DEVICE);

337
		/* clear tx_buffer data */
338
339
		tx_buffer->skb = NULL;
		dma_unmap_len_set(tx_buffer, len, 0);
340

341
342
343
344
		/* unmap remaining buffers */
		while (tx_desc != eop_desc) {
			tx_buffer++;
			tx_desc++;
345
			i++;
346
347
348
349
350
			if (unlikely(!i)) {
				i -= tx_ring->count;
				tx_buffer = tx_ring->tx_buffer_info;
				tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
			}
351

352
353
354
355
356
357
358
359
			/* unmap any remaining paged data */
			if (dma_unmap_len(tx_buffer, len)) {
				dma_unmap_page(tx_ring->dev,
					       dma_unmap_addr(tx_buffer, dma),
					       dma_unmap_len(tx_buffer, len),
					       DMA_TO_DEVICE);
				dma_unmap_len_set(tx_buffer, len, 0);
			}
360
361
		}

362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
		/* move us one more past the eop_desc for start of next pkt */
		tx_buffer++;
		tx_desc++;
		i++;
		if (unlikely(!i)) {
			i -= tx_ring->count;
			tx_buffer = tx_ring->tx_buffer_info;
			tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
		}

		/* issue prefetch for next Tx descriptor */
		prefetch(tx_desc);

		/* update budget accounting */
		budget--;
	} while (likely(budget));

	i += tx_ring->count;
380
	tx_ring->next_to_clean = i;
381
382
383
384
385
386
	u64_stats_update_begin(&tx_ring->syncp);
	tx_ring->stats.bytes += total_bytes;
	tx_ring->stats.packets += total_packets;
	u64_stats_update_end(&tx_ring->syncp);
	q_vector->tx.total_bytes += total_bytes;
	q_vector->tx.total_packets += total_packets;
387

388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
	if (check_for_tx_hang(tx_ring) && ixgbevf_check_tx_hang(tx_ring)) {
		struct ixgbe_hw *hw = &adapter->hw;
		union ixgbe_adv_tx_desc *eop_desc;

		eop_desc = tx_ring->tx_buffer_info[i].next_to_watch;

		pr_err("Detected Tx Unit Hang\n"
		       "  Tx Queue             <%d>\n"
		       "  TDH, TDT             <%x>, <%x>\n"
		       "  next_to_use          <%x>\n"
		       "  next_to_clean        <%x>\n"
		       "tx_buffer_info[next_to_clean]\n"
		       "  next_to_watch        <%p>\n"
		       "  eop_desc->wb.status  <%x>\n"
		       "  time_stamp           <%lx>\n"
		       "  jiffies              <%lx>\n",
		       tx_ring->queue_index,
		       IXGBE_READ_REG(hw, IXGBE_VFTDH(tx_ring->reg_idx)),
		       IXGBE_READ_REG(hw, IXGBE_VFTDT(tx_ring->reg_idx)),
		       tx_ring->next_to_use, i,
		       eop_desc, (eop_desc ? eop_desc->wb.status : 0),
		       tx_ring->tx_buffer_info[i].time_stamp, jiffies);

		netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);

		/* schedule immediate reset if we believe we hung */
414
		ixgbevf_tx_timeout_reset(adapter);
415
416
417
418

		return true;
	}

419
#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
420
	if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
421
		     (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
422
423
424
425
		/* Make sure that anybody stopping the queue after this
		 * sees the new next_to_clean.
		 */
		smp_mb();
426

427
428
		if (__netif_subqueue_stopped(tx_ring->netdev,
					     tx_ring->queue_index) &&
429
		    !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
430
431
			netif_wake_subqueue(tx_ring->netdev,
					    tx_ring->queue_index);
432
			++tx_ring->tx_stats.restart_queue;
433
434
435
		}
	}

436
	return !!budget;
437
438
}

Jacob Keller's avatar
Jacob Keller committed
439
440
441
442
443
444
/**
 * ixgbevf_rx_skb - Helper function to determine proper Rx method
 * @q_vector: structure containing interrupt and ring information
 * @skb: packet to send up
 **/
static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector,
445
			   struct sk_buff *skb)
Jacob Keller's avatar
Jacob Keller committed
446
{
447
448
449
450
451
452
453
454
455
#ifdef CONFIG_NET_RX_BUSY_POLL
	skb_mark_napi_id(skb, &q_vector->napi);

	if (ixgbevf_qv_busy_polling(q_vector)) {
		netif_receive_skb(skb);
		/* exit early if we busy polled */
		return;
	}
#endif /* CONFIG_NET_RX_BUSY_POLL */
Emil Tantilov's avatar
Emil Tantilov committed
456
457

	napi_gro_receive(&q_vector->napi, skb);
Jacob Keller's avatar
Jacob Keller committed
458
459
}

460
461
/**
 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
462
463
 * @ring: structure containig ring specific data
 * @rx_desc: current Rx descriptor being processed
464
 * @skb: skb currently being received and modified
465
 **/
466
static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring,
467
468
				       union ixgbe_adv_rx_desc *rx_desc,
				       struct sk_buff *skb)
469
{
470
	skb_checksum_none_assert(skb);
471
472

	/* Rx csum disabled */
473
	if (!(ring->netdev->features & NETIF_F_RXCSUM))
474
475
476
		return;

	/* if IP and error */
477
478
	if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&
	    ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {
479
		ring->rx_stats.csum_err++;
480
481
482
		return;
	}

483
	if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))
484
485
		return;

486
	if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {
487
		ring->rx_stats.csum_err++;
488
489
490
491
492
493
494
		return;
	}

	/* It must be a TCP or UDP packet with a valid checksum */
	skb->ip_summed = CHECKSUM_UNNECESSARY;
}

495
496
/**
 * ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor
497
498
499
500
501
502
503
 * @rx_ring: rx descriptor ring packet is being transacted on
 * @rx_desc: pointer to the EOP Rx descriptor
 * @skb: pointer to current skb being populated
 *
 * This function checks the ring, descriptor, and packet information in
 * order to populate the checksum, VLAN, protocol, and other fields within
 * the skb.
504
 **/
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring,
				       union ixgbe_adv_rx_desc *rx_desc,
				       struct sk_buff *skb)
{
	ixgbevf_rx_checksum(rx_ring, rx_desc, skb);

	if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {
		u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
		unsigned long *active_vlans = netdev_priv(rx_ring->netdev);

		if (test_bit(vid & VLAN_VID_MASK, active_vlans))
			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
	}

	skb->protocol = eth_type_trans(skb, rx_ring->netdev);
}

522
523
524
525
526
527
528
529
530
531
532
533
/**
 * ixgbevf_is_non_eop - process handling of non-EOP buffers
 * @rx_ring: Rx ring being processed
 * @rx_desc: Rx descriptor for current buffer
 * @skb: current socket buffer containing buffer in progress
 *
 * This function updates next to clean.  If the buffer is an EOP buffer
 * this function exits returning false, otherwise it will place the
 * sk_buff in the next buffer to be chained and return true indicating
 * that this is in fact a non-EOP buffer.
 **/
static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring,
534
			       union ixgbe_adv_rx_desc *rx_desc)
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
{
	u32 ntc = rx_ring->next_to_clean + 1;

	/* fetch, update, and store next to clean */
	ntc = (ntc < rx_ring->count) ? ntc : 0;
	rx_ring->next_to_clean = ntc;

	prefetch(IXGBEVF_RX_DESC(rx_ring, ntc));

	if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
		return false;

	return true;
}

550
551
static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring,
				      struct ixgbevf_rx_buffer *bi)
552
{
553
	struct page *page = bi->page;
554
555
	dma_addr_t dma = bi->dma;

556
557
	/* since we are recycling buffers we should seldom need to alloc */
	if (likely(page))
558
559
		return true;

560
561
562
563
	/* alloc new page for storage */
	page = dev_alloc_page();
	if (unlikely(!page)) {
		rx_ring->rx_stats.alloc_rx_page_failed++;
564
565
566
		return false;
	}

567
568
569
	/* map page for use */
	dma = dma_map_page(rx_ring->dev, page, 0,
			   PAGE_SIZE, DMA_FROM_DEVICE);
570
571
572
573
574

	/* if mapping failed free memory back to system since
	 * there isn't much point in holding memory we can't use
	 */
	if (dma_mapping_error(rx_ring->dev, dma)) {
575
		__free_page(page);
576
577
578
579
580
581

		rx_ring->rx_stats.alloc_rx_buff_failed++;
		return false;
	}

	bi->dma = dma;
582
583
	bi->page = page;
	bi->page_offset = 0;
584
585
586
587

	return true;
}

588
589
/**
 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
590
 * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on
591
 * @cleaned_count: number of buffers to replace
592
 **/
593
static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring,
594
				     u16 cleaned_count)
595
596
597
{
	union ixgbe_adv_rx_desc *rx_desc;
	struct ixgbevf_rx_buffer *bi;
598
	unsigned int i = rx_ring->next_to_use;
599

600
601
602
	/* nothing to do or no valid netdev defined */
	if (!cleaned_count || !rx_ring->netdev)
		return;
603

604
605
606
	rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
	bi = &rx_ring->rx_buffer_info[i];
	i -= rx_ring->count;
607

608
	do {
609
		if (!ixgbevf_alloc_mapped_page(rx_ring, bi))
610
			break;
611

612
613
614
		/* Refresh the desc even if pkt_addr didn't change
		 * because each write-back erases this info.
		 */
615
		rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
616

617
618
		rx_desc++;
		bi++;
619
		i++;
620
621
622
623
624
625
626
627
628
629
630
631
632
		if (unlikely(!i)) {
			rx_desc = IXGBEVF_RX_DESC(rx_ring, 0);
			bi = rx_ring->rx_buffer_info;
			i -= rx_ring->count;
		}

		/* clear the hdr_addr for the next_to_use descriptor */
		rx_desc->read.hdr_addr = 0;

		cleaned_count--;
	} while (cleaned_count);

	i += rx_ring->count;
633

634
635
636
637
	if (rx_ring->next_to_use != i) {
		/* record the next descriptor to use */
		rx_ring->next_to_use = i;

638
639
640
		/* update next to alloc since we have filled the ring */
		rx_ring->next_to_alloc = i;

641
642
643
644
645
646
647
648
		/* Force memory writes to complete before letting h/w
		 * know there are new descriptors to fetch.  (Only
		 * applicable for weak-ordered memory model archs,
		 * such as IA-64).
		 */
		wmb();
		ixgbevf_write_tail(rx_ring, i);
	}
649
650
}

651
652
/**
 * ixgbevf_pull_tail - ixgbevf specific version of skb_pull_tail
653
654
655
656
657
658
659
660
661
 * @rx_ring: rx descriptor ring packet is being transacted on
 * @skb: pointer to current skb being adjusted
 *
 * This function is an ixgbevf specific version of __pskb_pull_tail.  The
 * main difference between this version and the original function is that
 * this function can make several assumptions about the state of things
 * that allow for significant optimizations versus the standard function.
 * As a result we can do things like drop a frag and maintain an accurate
 * truesize for the skb.
662
 **/
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
static void ixgbevf_pull_tail(struct ixgbevf_ring *rx_ring,
			      struct sk_buff *skb)
{
	struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
	unsigned char *va;
	unsigned int pull_len;

	/* it is valid to use page_address instead of kmap since we are
	 * working with pages allocated out of the lomem pool per
	 * alloc_page(GFP_ATOMIC)
	 */
	va = skb_frag_address(frag);

	/* we need the header to contain the greater of either ETH_HLEN or
	 * 60 bytes if the skb->len is less than 60 for skb_pad.
	 */
	pull_len = eth_get_headlen(va, IXGBEVF_RX_HDR_SIZE);

	/* align pull length to size of long to optimize memcpy performance */
	skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));

	/* update all of the pointers */
	skb_frag_size_sub(frag, pull_len);
	frag->page_offset += pull_len;
	skb->data_len -= pull_len;
	skb->tail += pull_len;
}

691
692
/**
 * ixgbevf_cleanup_headers - Correct corrupted or empty headers
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
 * @rx_ring: rx descriptor ring packet is being transacted on
 * @rx_desc: pointer to the EOP Rx descriptor
 * @skb: pointer to current skb being fixed
 *
 * Check for corrupted packet headers caused by senders on the local L2
 * embedded NIC switch not setting up their Tx Descriptors right.  These
 * should be very rare.
 *
 * Also address the case where we are pulling data in on pages only
 * and as such no data is present in the skb header.
 *
 * In addition if skb is not at least 60 bytes we need to pad it so that
 * it is large enough to qualify as a valid Ethernet frame.
 *
 * Returns true if an error was encountered and skb was freed.
708
 **/
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring,
				    union ixgbe_adv_rx_desc *rx_desc,
				    struct sk_buff *skb)
{
	/* verify that the packet does not have any known errors */
	if (unlikely(ixgbevf_test_staterr(rx_desc,
					  IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) {
		struct net_device *netdev = rx_ring->netdev;

		if (!(netdev->features & NETIF_F_RXALL)) {
			dev_kfree_skb_any(skb);
			return true;
		}
	}

	/* place header in linear portion of buffer */
	if (skb_is_nonlinear(skb))
		ixgbevf_pull_tail(rx_ring, skb);

728
729
730
	/* if eth_skb_pad returns an error the skb was freed */
	if (eth_skb_pad(skb))
		return true;
731
732
733
734

	return false;
}

735
736
/**
 * ixgbevf_reuse_rx_page - page flip buffer and store it back on the ring
737
738
739
740
 * @rx_ring: rx descriptor ring to store buffers on
 * @old_buff: donor buffer to have page reused
 *
 * Synchronizes page for reuse by the adapter
741
 **/
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring,
				  struct ixgbevf_rx_buffer *old_buff)
{
	struct ixgbevf_rx_buffer *new_buff;
	u16 nta = rx_ring->next_to_alloc;

	new_buff = &rx_ring->rx_buffer_info[nta];

	/* update, and store next to alloc */
	nta++;
	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;

	/* transfer page from old buffer to new buffer */
	new_buff->page = old_buff->page;
	new_buff->dma = old_buff->dma;
	new_buff->page_offset = old_buff->page_offset;

	/* sync the buffer for use by the device */
	dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma,
					 new_buff->page_offset,
					 IXGBEVF_RX_BUFSZ,
					 DMA_FROM_DEVICE);
}

static inline bool ixgbevf_page_is_reserved(struct page *page)
{
768
	return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
769
770
}

771
772
/**
 * ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff
773
774
775
776
777
778
779
780
781
782
783
784
 * @rx_ring: rx descriptor ring to transact packets on
 * @rx_buffer: buffer containing page to add
 * @rx_desc: descriptor containing length of buffer written by hardware
 * @skb: sk_buff to place the data into
 *
 * This function will add the data contained in rx_buffer->page to the skb.
 * This is done either through a direct copy if the data in the buffer is
 * less than the skb header size, otherwise it will just attach the page as
 * a frag to the skb.
 *
 * The function will then update the page offset if necessary and return
 * true if the buffer can be reused by the adapter.
785
 **/
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
static bool ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring,
				struct ixgbevf_rx_buffer *rx_buffer,
				union ixgbe_adv_rx_desc *rx_desc,
				struct sk_buff *skb)
{
	struct page *page = rx_buffer->page;
	unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
#if (PAGE_SIZE < 8192)
	unsigned int truesize = IXGBEVF_RX_BUFSZ;
#else
	unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
#endif

	if ((size <= IXGBEVF_RX_HDR_SIZE) && !skb_is_nonlinear(skb)) {
		unsigned char *va = page_address(page) + rx_buffer->page_offset;

		memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));

		/* page is not reserved, we can reuse buffer as is */
		if (likely(!ixgbevf_page_is_reserved(page)))
			return true;

		/* this page cannot be reused so discard it */
		put_page(page);
		return false;
	}

	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
			rx_buffer->page_offset, size, truesize);

	/* avoid re-using remote pages */
	if (unlikely(ixgbevf_page_is_reserved(page)))
		return false;

#if (PAGE_SIZE < 8192)
	/* if we are only owner of page we can reuse it */
	if (unlikely(page_count(page) != 1))
		return false;

	/* flip page offset to other buffer */
	rx_buffer->page_offset ^= IXGBEVF_RX_BUFSZ;

#else
	/* move offset up to the next cache line */
	rx_buffer->page_offset += truesize;

	if (rx_buffer->page_offset > (PAGE_SIZE - IXGBEVF_RX_BUFSZ))
		return false;

#endif
	/* Even if we own the page, we are not allowed to use atomic_set()
	 * This would break get_page_unless_zero() users.
	 */
	atomic_inc(&page->_count);

	return true;
}

static struct sk_buff *ixgbevf_fetch_rx_buffer(struct ixgbevf_ring *rx_ring,
					       union ixgbe_adv_rx_desc *rx_desc,
					       struct sk_buff *skb)
{
	struct ixgbevf_rx_buffer *rx_buffer;
	struct page *page;

	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
	page = rx_buffer->page;
	prefetchw(page);

	if (likely(!skb)) {
		void *page_addr = page_address(page) +
				  rx_buffer->page_offset;

		/* prefetch first cache line of first page */
		prefetch(page_addr);
#if L1_CACHE_BYTES < 128
		prefetch(page_addr + L1_CACHE_BYTES);
#endif

		/* allocate a skb to store the frags */
		skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
						IXGBEVF_RX_HDR_SIZE);
		if (unlikely(!skb)) {
			rx_ring->rx_stats.alloc_rx_buff_failed++;
			return NULL;
		}

		/* we will be copying header into skb->data in
		 * pskb_may_pull so it is in our interest to prefetch
		 * it now to avoid a possible cache miss
		 */
		prefetchw(skb->data);
	}

	/* we are reusing so sync this buffer for CPU use */
	dma_sync_single_range_for_cpu(rx_ring->dev,
				      rx_buffer->dma,
				      rx_buffer->page_offset,
				      IXGBEVF_RX_BUFSZ,
				      DMA_FROM_DEVICE);

	/* pull page into skb */
	if (ixgbevf_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) {
		/* hand second half of page back to the ring */
		ixgbevf_reuse_rx_page(rx_ring, rx_buffer);
	} else {
		/* we are not reusing the buffer so unmap it */
		dma_unmap_page(rx_ring->dev, rx_buffer->dma,
			       PAGE_SIZE, DMA_FROM_DEVICE);
	}

	/* clear contents of buffer_info */
	rx_buffer->dma = 0;
	rx_buffer->page = NULL;

	return skb;
}

904
static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
905
					     u32 qmask)
906
907
908
{
	struct ixgbe_hw *hw = &adapter->hw;

909
	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
910
911
}

912
913
914
static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
				struct ixgbevf_ring *rx_ring,
				int budget)
915
916
{
	unsigned int total_rx_bytes = 0, total_rx_packets = 0;
917
	u16 cleaned_count = ixgbevf_desc_unused(rx_ring);
918
	struct sk_buff *skb = rx_ring->skb;
919

920
	while (likely(total_rx_packets < budget)) {
921
		union ixgbe_adv_rx_desc *rx_desc;
922

923
924
925
926
927
928
		/* return some buffers to hardware, one at a time is too slow */
		if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
			ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
			cleaned_count = 0;
		}

929
		rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean);
930
931

		if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_DD))
932
933
			break;

934
935
936
937
938
		/* This memory barrier is needed to keep us from reading
		 * any other fields out of the rx_desc until we know the
		 * RXD_STAT_DD bit is set
		 */
		rmb();
939

940
941
		/* retrieve a buffer from the ring */
		skb = ixgbevf_fetch_rx_buffer(rx_ring, rx_desc, skb);
942

943
944
945
		/* exit if we failed to retrieve a buffer */
		if (!skb)
			break;
946

947
948
		cleaned_count++;

949
950
		/* fetch next buffer in frame if non-eop */
		if (ixgbevf_is_non_eop(rx_ring, rx_desc))
951
			continue;
952

953
954
955
		/* verify the packet layout is correct */
		if (ixgbevf_cleanup_headers(rx_ring, rx_desc, skb)) {
			skb = NULL;
956
			continue;
957
958
959
960
961
		}

		/* probably a little skewed due to removing CRC */
		total_rx_bytes += skb->len;

962
963
964
		/* Workaround hardware that can't do proper VEPA multicast
		 * source pruning.
		 */
965
		if ((skb->pkt_type == PACKET_BROADCAST ||
966
		     skb->pkt_type == PACKET_MULTICAST) &&
967
		    ether_addr_equal(rx_ring->netdev->dev_addr,
968
				     eth_hdr(skb)->h_source)) {
969
			dev_kfree_skb_irq(skb);
970
			continue;
971
972
		}

973
974
975
976
		/* populate checksum, VLAN, and protocol */
		ixgbevf_process_skb_fields(rx_ring, rx_desc, skb);

		ixgbevf_rx_skb(q_vector, skb);
977

978
979
980
		/* reset skb pointer */
		skb = NULL;

981
		/* update budget accounting */
982
983
		total_rx_packets++;
	}
984

985
986
987
	/* place incomplete frames back on ring for completion */
	rx_ring->skb = skb;

988
	u64_stats_update_begin(&rx_ring->syncp);
989
990
	rx_ring->stats.packets += total_rx_packets;
	rx_ring->stats.bytes += total_rx_bytes;
991
	u64_stats_update_end(&rx_ring->syncp);
992
993
	q_vector->rx.total_packets += total_rx_packets;
	q_vector->rx.total_bytes += total_rx_bytes;
994

995
	return total_rx_packets;
996
997
998
}

/**
999
 * ixgbevf_poll - NAPI polling calback
1000
1001
1002
 * @napi: napi struct with our devices info in it
 * @budget: amount of work driver is allowed to do this pass, in packets
 *
1003
 * This function will clean more than one or more rings associated with a
1004
1005
 * q_vector.
 **/
1006
static int ixgbevf_poll(struct napi_struct *napi, int budget)
1007
1008
1009
1010
{
	struct ixgbevf_q_vector *q_vector =
		container_of(napi, struct ixgbevf_q_vector, napi);
	struct ixgbevf_adapter *adapter = q_vector->adapter;
1011
1012
1013
1014
1015
1016
	struct ixgbevf_ring *ring;
	int per_ring_budget;
	bool clean_complete = true;

	ixgbevf_for_each_ring(ring, q_vector->tx)
		clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring);
1017

1018
1019
1020
1021
1022
#ifdef CONFIG_NET_RX_BUSY_POLL
	if (!ixgbevf_qv_lock_napi(q_vector))
		return budget;
#endif

1023
	/* attempt to distribute budget to each queue fairly, but don't allow
1024
1025
	 * the budget to go below 1 because we'll exit polling
	 */
1026
1027
1028
1029
1030
1031
	if (q_vector->rx.count > 1)
		per_ring_budget = max(budget/q_vector->rx.count, 1);
	else
		per_ring_budget = budget;

	ixgbevf_for_each_ring(ring, q_vector->rx)
1032
1033
1034
		clean_complete &= (ixgbevf_clean_rx_irq(q_vector, ring,
							per_ring_budget)
				   < per_ring_budget);
1035

1036
1037
1038
1039
#ifdef CONFIG_NET_RX_BUSY_POLL
	ixgbevf_qv_unlock_napi(q_vector);
#endif

1040
1041
1042
1043
1044
1045
1046
	/* If all work not completed, return budget and keep polling */
	if (!clean_complete)
		return budget;
	/* all work done, exit the polling mode */
	napi_complete(napi);
	if (adapter->rx_itr_setting & 1)
		ixgbevf_set_itr(q_vector);
1047
1048
	if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
	    !test_bit(__IXGBEVF_REMOVING, &adapter->state))
1049
1050
		ixgbevf_irq_enable_queues(adapter,
					  1 << q_vector->v_idx);
1051

1052
	return 0;
1053
1054
}

1055
1056
1057
/**
 * ixgbevf_write_eitr - write VTEITR register in hardware specific way
 * @q_vector: structure containing interrupt and ring information
1058
 **/
1059
void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector)
1060
1061
1062
1063
1064
1065
{
	struct ixgbevf_adapter *adapter = q_vector->adapter;
	struct ixgbe_hw *hw = &adapter->hw;
	int v_idx = q_vector->v_idx;
	u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;

1066
	/* set the WDIS bit to not clear the timer bits and cause an
1067
1068
1069
1070
1071
1072
	 * immediate assertion of the interrupt
	 */
	itr_reg |= IXGBE_EITR_CNT_WDIS;

	IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
}
1073

1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
#ifdef CONFIG_NET_RX_BUSY_POLL
/* must be called with local_bh_disable()d */
static int ixgbevf_busy_poll_recv(struct napi_struct *napi)
{
	struct ixgbevf_q_vector *q_vector =
			container_of(napi, struct ixgbevf_q_vector, napi);
	struct ixgbevf_adapter *adapter = q_vector->adapter;
	struct ixgbevf_ring  *ring;
	int found = 0;

	if (test_bit(__IXGBEVF_DOWN, &adapter->state))
		return LL_FLUSH_FAILED;

	if (!ixgbevf_qv_lock_poll(q_vector))
		return LL_FLUSH_BUSY;

	ixgbevf_for_each_ring(ring, q_vector->rx) {
		found = ixgbevf_clean_rx_irq(q_vector, ring, 4);
1092
1093
#ifdef BP_EXTENDED_STATS
		if (found)
1094
			ring->stats.cleaned += found;
1095
		else
1096
			ring->stats.misses++;
1097
#endif
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
		if (found)
			break;
	}

	ixgbevf_qv_unlock_poll(q_vector);

	return found;
}
#endif /* CONFIG_NET_RX_BUSY_POLL */

1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
/**
 * ixgbevf_configure_msix - Configure MSI-X hardware
 * @adapter: board private structure
 *
 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
 * interrupts.
 **/
static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
{
	struct ixgbevf_q_vector *q_vector;
1118
	int q_vectors, v_idx;
1119
1120

	q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1121
	adapter->eims_enable_mask = 0;
1122

1123
	/* Populate the IVAR table and set the ITR values to the
1124
1125
1126
	 * corresponding register.
	 */
	for (v_idx = 0; v_idx < q_vectors; v_idx++) {
1127
		struct ixgbevf_ring *ring;
1128

1129
		q_vector = adapter->q_vector[v_idx];
1130
1131
1132
1133
1134
1135

		ixgbevf_for_each_ring(ring, q_vector->rx)
			ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx);

		ixgbevf_for_each_ring(ring, q_vector->tx)
			ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx);
1136

1137
		if (q_vector->tx.ring && !q_vector->rx.ring) {
1138
			/* Tx only vector */
1139
1140
1141
1142
1143
			if (adapter->tx_itr_setting == 1)
				q_vector->itr = IXGBE_10K_ITR;
			else
				q_vector->itr = adapter->tx_itr_setting;
		} else {
1144
			/* Rx or Rx/Tx vector */
1145
1146
1147
1148
1149
1150
1151
1152
			if (adapter->rx_itr_setting == 1)
				q_vector->itr = IXGBE_20K_ITR;
			else
				q_vector->itr = adapter->rx_itr_setting;
		}

		/* add q_vector eims value to global eims_enable_mask */
		adapter->eims_enable_mask |= 1 << v_idx;
1153

1154
		ixgbevf_write_eitr(q_vector);
1155
1156
1157
	}

	ixgbevf_set_ivar(adapter, -1, 1, v_idx);
1158
1159
1160
	/* setup eims_other and add value to global eims_enable_mask */
	adapter->eims_other = 1 << v_idx;
	adapter->eims_enable_mask |= adapter->eims_other;
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
}

enum latency_range {
	lowest_latency = 0,
	low_latency = 1,
	bulk_latency = 2,
	latency_invalid = 255
};

/**
 * ixgbevf_update_itr - update the dynamic ITR value based on statistics
1172
1173
 * @q_vector: structure containing interrupt and ring information
 * @ring_container: structure containing ring performance data
1174
 *
1175
1176
1177
1178
1179
1180
1181
 * Stores a new ITR value based on packets and byte
 * counts during the last interrupt.  The advantage of per interrupt
 * computation is faster updates and more accurate ITR for the current
 * traffic pattern.  Constants in this function were computed
 * based on theoretical maximum wire speed and thresholds were set based
 * on testing data as well as attempting to minimize response time
 * while increasing bulk throughput.
1182
 **/
1183
1184
static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector,
			       struct ixgbevf_ring_container *ring_container)
1185
{
1186
1187
	int bytes = ring_container->total_bytes;
	int packets = ring_container->total_packets;
1188
1189
	u32 timepassed_us;
	u64 bytes_perint;
1190
	u8 itr_setting = ring_container->itr;
1191
1192

	if (packets == 0)
1193
		return;
1194

1195
	/* simple throttle rate management
1196
1197
1198
1199
1200
	 *    0-20MB/s lowest (100000 ints/s)
	 *   20-100MB/s low   (20000 ints/s)
	 *  100-1249MB/s bulk (8000 ints/s)
	 */
	/* what was last interrupt timeslice? */
1201
	timepassed_us = q_vector->itr >> 2;
1202
1203
1204
1205
	bytes_perint = bytes / timepassed_us; /* bytes/usec */

	switch (itr_setting) {
	case lowest_latency:
1206
		if (bytes_perint > 10)
1207
			itr_setting = low_latency;
1208
1209
		break;
	case low_latency:
1210
		if (bytes_perint > 20)
1211
			itr_setting = bulk_latency;
1212
		else if (bytes_perint <= 10)
1213
			itr_setting = lowest_latency;
1214
1215
		break;
	case bulk_latency:
1216
		if (bytes_perint <= 20)
1217
			itr_setting = low_latency;
1218
1219
1220
		break;
	}

1221
1222
1223
1224
1225
1226
	/* clear work counters since we have the values we need */
	ring_container->total_bytes = 0;
	ring_container->total_packets = 0;

	/* write updated itr to ring container */
	ring_container->itr = itr_setting;
1227
1228
}

1229
static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector)
1230
{
1231
1232
	u32 new_itr = q_vector->itr;
	u8 current_itr;
1233

1234
1235
	ixgbevf_update_itr(q_vector, &q_vector->tx);
	ixgbevf_update_itr(q_vector, &q_vector->rx);
1236

1237
	current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
1238
1239
1240
1241

	switch (current_itr) {
	/* counts and packets in update_itr are dependent on these numbers */
	case lowest_latency:
1242
		new_itr = IXGBE_100K_ITR;
1243
1244
		break;
	case low_latency:
1245
		new_itr = IXGBE_20K_ITR;
1246
1247
1248
		break;
	case bulk_latency:
	default:
1249
		new_itr = IXGBE_8K_ITR;
1250
1251
1252
		break;
	}

1253
	if (new_itr != q_vector->itr) {
1254
		/* do an exponential smoothing */
1255
1256
1257
1258
1259
1260
1261
		new_itr = (10 * new_itr * q_vector->itr) /
			  ((9 * new_itr) + q_vector->itr);

		/* save the algorithm value here */
		q_vector->itr = new_itr;

		ixgbevf_write_eitr(q_vector);
1262
1263
1264
	}
}

1265
static irqreturn_t ixgbevf_msix_other(int irq, void *data)
1266
{
1267
	struct ixgbevf_adapter *adapter = data;
1268
	struct ixgbe_hw *hw = &adapter->hw;
1269

1270
	hw->mac.get_link_status = 1;
1271

1272
	ixgbevf_service_event_schedule(adapter);
1273

1274
1275
	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other);

1276
1277
1278
1279
	return IRQ_HANDLED;
}

/**