Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1 | /**************************************************************************** |
| 2 | * Driver for Solarflare Solarstorm network controllers and boards |
| 3 | * Copyright 2005-2006 Fen Systems Ltd. |
Ben Hutchings | 0a6f40c | 2011-02-25 00:01:34 +0000 | [diff] [blame] | 4 | * Copyright 2006-2011 Solarflare Communications Inc. |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify it |
| 7 | * under the terms of the GNU General Public License version 2 as published |
| 8 | * by the Free Software Foundation, incorporated herein by reference. |
| 9 | */ |
| 10 | |
| 11 | #include <linux/bitops.h> |
| 12 | #include <linux/delay.h> |
Alexey Dobriyan | a6b7a40 | 2011-06-06 10:43:46 +0000 | [diff] [blame] | 13 | #include <linux/interrupt.h> |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 14 | #include <linux/pci.h> |
| 15 | #include <linux/module.h> |
| 16 | #include <linux/seq_file.h> |
| 17 | #include "net_driver.h" |
| 18 | #include "bitfield.h" |
| 19 | #include "efx.h" |
| 20 | #include "nic.h" |
| 21 | #include "regs.h" |
| 22 | #include "io.h" |
| 23 | #include "workarounds.h" |
| 24 | |
| 25 | /************************************************************************** |
| 26 | * |
| 27 | * Configurable values |
| 28 | * |
| 29 | ************************************************************************** |
| 30 | */ |
| 31 | |
| 32 | /* This is set to 16 for a good reason. In summary, if larger than |
| 33 | * 16, the descriptor cache holds more than a default socket |
| 34 | * buffer's worth of packets (for UDP we can only have at most one |
| 35 | * socket buffer's worth outstanding). This combined with the fact |
| 36 | * that we only get 1 TX event per descriptor cache means the NIC |
| 37 | * goes idle. |
| 38 | */ |
| 39 | #define TX_DC_ENTRIES 16 |
| 40 | #define TX_DC_ENTRIES_ORDER 1 |
| 41 | |
| 42 | #define RX_DC_ENTRIES 64 |
| 43 | #define RX_DC_ENTRIES_ORDER 3 |
| 44 | |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 45 | /* If EFX_MAX_INT_ERRORS internal errors occur within |
| 46 | * EFX_INT_ERROR_EXPIRE seconds, we consider the NIC broken and |
| 47 | * disable it. |
| 48 | */ |
| 49 | #define EFX_INT_ERROR_EXPIRE 3600 |
| 50 | #define EFX_MAX_INT_ERRORS 5 |
| 51 | |
| 52 | /* We poll for events every FLUSH_INTERVAL ms, and check FLUSH_POLL_COUNT times |
| 53 | */ |
| 54 | #define EFX_FLUSH_INTERVAL 10 |
| 55 | #define EFX_FLUSH_POLL_COUNT 100 |
| 56 | |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 57 | /* Depth of RX flush request fifo */ |
| 58 | #define EFX_RX_FLUSH_COUNT 4 |
| 59 | |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 60 | /* Generated event code for efx_generate_test_event() */ |
| 61 | #define EFX_CHANNEL_MAGIC_TEST(_channel) \ |
Steve Hodgson | d730dc5 | 2010-06-01 11:19:09 +0000 | [diff] [blame] | 62 | (0x00010100 + (_channel)->channel) |
| 63 | |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 64 | /* Generated event code for efx_generate_fill_event() */ |
| 65 | #define EFX_CHANNEL_MAGIC_FILL(_channel) \ |
| 66 | (0x00010200 + (_channel)->channel) |
| 67 | |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 68 | /************************************************************************** |
| 69 | * |
| 70 | * Solarstorm hardware access |
| 71 | * |
| 72 | **************************************************************************/ |
| 73 | |
| 74 | static inline void efx_write_buf_tbl(struct efx_nic *efx, efx_qword_t *value, |
| 75 | unsigned int index) |
| 76 | { |
| 77 | efx_sram_writeq(efx, efx->membase + efx->type->buf_tbl_base, |
| 78 | value, index); |
| 79 | } |
| 80 | |
| 81 | /* Read the current event from the event queue */ |
| 82 | static inline efx_qword_t *efx_event(struct efx_channel *channel, |
| 83 | unsigned int index) |
| 84 | { |
Ben Hutchings | d4fabcc | 2011-04-04 14:22:11 +0100 | [diff] [blame] | 85 | return ((efx_qword_t *) (channel->eventq.addr)) + |
| 86 | (index & channel->eventq_mask); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 87 | } |
| 88 | |
| 89 | /* See if an event is present |
| 90 | * |
| 91 | * We check both the high and low dword of the event for all ones. We |
| 92 | * wrote all ones when we cleared the event, and no valid event can |
| 93 | * have all ones in either its high or low dwords. This approach is |
| 94 | * robust against reordering. |
| 95 | * |
| 96 | * Note that using a single 64-bit comparison is incorrect; even |
| 97 | * though the CPU read will be atomic, the DMA write may not be. |
| 98 | */ |
| 99 | static inline int efx_event_present(efx_qword_t *event) |
| 100 | { |
Eric Dumazet | 807540b | 2010-09-23 05:40:09 +0000 | [diff] [blame] | 101 | return !(EFX_DWORD_IS_ALL_ONES(event->dword[0]) | |
| 102 | EFX_DWORD_IS_ALL_ONES(event->dword[1])); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 103 | } |
| 104 | |
| 105 | static bool efx_masked_compare_oword(const efx_oword_t *a, const efx_oword_t *b, |
| 106 | const efx_oword_t *mask) |
| 107 | { |
| 108 | return ((a->u64[0] ^ b->u64[0]) & mask->u64[0]) || |
| 109 | ((a->u64[1] ^ b->u64[1]) & mask->u64[1]); |
| 110 | } |
| 111 | |
| 112 | int efx_nic_test_registers(struct efx_nic *efx, |
| 113 | const struct efx_nic_register_test *regs, |
| 114 | size_t n_regs) |
| 115 | { |
| 116 | unsigned address = 0, i, j; |
| 117 | efx_oword_t mask, imask, original, reg, buf; |
| 118 | |
| 119 | /* Falcon should be in loopback to isolate the XMAC from the PHY */ |
| 120 | WARN_ON(!LOOPBACK_INTERNAL(efx)); |
| 121 | |
| 122 | for (i = 0; i < n_regs; ++i) { |
| 123 | address = regs[i].address; |
| 124 | mask = imask = regs[i].mask; |
| 125 | EFX_INVERT_OWORD(imask); |
| 126 | |
| 127 | efx_reado(efx, &original, address); |
| 128 | |
| 129 | /* bit sweep on and off */ |
| 130 | for (j = 0; j < 128; j++) { |
| 131 | if (!EFX_EXTRACT_OWORD32(mask, j, j)) |
| 132 | continue; |
| 133 | |
| 134 | /* Test this testable bit can be set in isolation */ |
| 135 | EFX_AND_OWORD(reg, original, mask); |
| 136 | EFX_SET_OWORD32(reg, j, j, 1); |
| 137 | |
| 138 | efx_writeo(efx, ®, address); |
| 139 | efx_reado(efx, &buf, address); |
| 140 | |
| 141 | if (efx_masked_compare_oword(®, &buf, &mask)) |
| 142 | goto fail; |
| 143 | |
| 144 | /* Test this testable bit can be cleared in isolation */ |
| 145 | EFX_OR_OWORD(reg, original, mask); |
| 146 | EFX_SET_OWORD32(reg, j, j, 0); |
| 147 | |
| 148 | efx_writeo(efx, ®, address); |
| 149 | efx_reado(efx, &buf, address); |
| 150 | |
| 151 | if (efx_masked_compare_oword(®, &buf, &mask)) |
| 152 | goto fail; |
| 153 | } |
| 154 | |
| 155 | efx_writeo(efx, &original, address); |
| 156 | } |
| 157 | |
| 158 | return 0; |
| 159 | |
| 160 | fail: |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 161 | netif_err(efx, hw, efx->net_dev, |
| 162 | "wrote "EFX_OWORD_FMT" read "EFX_OWORD_FMT |
| 163 | " at address 0x%x mask "EFX_OWORD_FMT"\n", EFX_OWORD_VAL(reg), |
| 164 | EFX_OWORD_VAL(buf), address, EFX_OWORD_VAL(mask)); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 165 | return -EIO; |
| 166 | } |
| 167 | |
| 168 | /************************************************************************** |
| 169 | * |
| 170 | * Special buffer handling |
| 171 | * Special buffers are used for event queues and the TX and RX |
| 172 | * descriptor rings. |
| 173 | * |
| 174 | *************************************************************************/ |
| 175 | |
| 176 | /* |
| 177 | * Initialise a special buffer |
| 178 | * |
| 179 | * This will define a buffer (previously allocated via |
| 180 | * efx_alloc_special_buffer()) in the buffer table, allowing |
| 181 | * it to be used for event queues, descriptor rings etc. |
| 182 | */ |
| 183 | static void |
| 184 | efx_init_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer) |
| 185 | { |
| 186 | efx_qword_t buf_desc; |
| 187 | int index; |
| 188 | dma_addr_t dma_addr; |
| 189 | int i; |
| 190 | |
| 191 | EFX_BUG_ON_PARANOID(!buffer->addr); |
| 192 | |
| 193 | /* Write buffer descriptors to NIC */ |
| 194 | for (i = 0; i < buffer->entries; i++) { |
| 195 | index = buffer->index + i; |
Ben Hutchings | 5b6262d | 2012-02-02 21:21:15 +0000 | [diff] [blame^] | 196 | dma_addr = buffer->dma_addr + (i * EFX_BUF_SIZE); |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 197 | netif_dbg(efx, probe, efx->net_dev, |
| 198 | "mapping special buffer %d at %llx\n", |
| 199 | index, (unsigned long long)dma_addr); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 200 | EFX_POPULATE_QWORD_3(buf_desc, |
| 201 | FRF_AZ_BUF_ADR_REGION, 0, |
| 202 | FRF_AZ_BUF_ADR_FBUF, dma_addr >> 12, |
| 203 | FRF_AZ_BUF_OWNER_ID_FBUF, 0); |
| 204 | efx_write_buf_tbl(efx, &buf_desc, index); |
| 205 | } |
| 206 | } |
| 207 | |
| 208 | /* Unmaps a buffer and clears the buffer table entries */ |
| 209 | static void |
| 210 | efx_fini_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer) |
| 211 | { |
| 212 | efx_oword_t buf_tbl_upd; |
| 213 | unsigned int start = buffer->index; |
| 214 | unsigned int end = (buffer->index + buffer->entries - 1); |
| 215 | |
| 216 | if (!buffer->entries) |
| 217 | return; |
| 218 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 219 | netif_dbg(efx, hw, efx->net_dev, "unmapping special buffers %d-%d\n", |
| 220 | buffer->index, buffer->index + buffer->entries - 1); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 221 | |
| 222 | EFX_POPULATE_OWORD_4(buf_tbl_upd, |
| 223 | FRF_AZ_BUF_UPD_CMD, 0, |
| 224 | FRF_AZ_BUF_CLR_CMD, 1, |
| 225 | FRF_AZ_BUF_CLR_END_ID, end, |
| 226 | FRF_AZ_BUF_CLR_START_ID, start); |
| 227 | efx_writeo(efx, &buf_tbl_upd, FR_AZ_BUF_TBL_UPD); |
| 228 | } |
| 229 | |
| 230 | /* |
| 231 | * Allocate a new special buffer |
| 232 | * |
| 233 | * This allocates memory for a new buffer, clears it and allocates a |
| 234 | * new buffer ID range. It does not write into the buffer table. |
| 235 | * |
| 236 | * This call will allocate 4KB buffers, since 8KB buffers can't be |
| 237 | * used for event queues and descriptor rings. |
| 238 | */ |
| 239 | static int efx_alloc_special_buffer(struct efx_nic *efx, |
| 240 | struct efx_special_buffer *buffer, |
| 241 | unsigned int len) |
| 242 | { |
| 243 | len = ALIGN(len, EFX_BUF_SIZE); |
| 244 | |
Ben Hutchings | 58758aa | 2010-09-10 06:41:26 +0000 | [diff] [blame] | 245 | buffer->addr = dma_alloc_coherent(&efx->pci_dev->dev, len, |
| 246 | &buffer->dma_addr, GFP_KERNEL); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 247 | if (!buffer->addr) |
| 248 | return -ENOMEM; |
| 249 | buffer->len = len; |
| 250 | buffer->entries = len / EFX_BUF_SIZE; |
| 251 | BUG_ON(buffer->dma_addr & (EFX_BUF_SIZE - 1)); |
| 252 | |
| 253 | /* All zeros is a potentially valid event so memset to 0xff */ |
| 254 | memset(buffer->addr, 0xff, len); |
| 255 | |
| 256 | /* Select new buffer ID */ |
| 257 | buffer->index = efx->next_buffer_table; |
| 258 | efx->next_buffer_table += buffer->entries; |
| 259 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 260 | netif_dbg(efx, probe, efx->net_dev, |
| 261 | "allocating special buffers %d-%d at %llx+%x " |
| 262 | "(virt %p phys %llx)\n", buffer->index, |
| 263 | buffer->index + buffer->entries - 1, |
| 264 | (u64)buffer->dma_addr, len, |
| 265 | buffer->addr, (u64)virt_to_phys(buffer->addr)); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 266 | |
| 267 | return 0; |
| 268 | } |
| 269 | |
| 270 | static void |
| 271 | efx_free_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer) |
| 272 | { |
| 273 | if (!buffer->addr) |
| 274 | return; |
| 275 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 276 | netif_dbg(efx, hw, efx->net_dev, |
| 277 | "deallocating special buffers %d-%d at %llx+%x " |
| 278 | "(virt %p phys %llx)\n", buffer->index, |
| 279 | buffer->index + buffer->entries - 1, |
| 280 | (u64)buffer->dma_addr, buffer->len, |
| 281 | buffer->addr, (u64)virt_to_phys(buffer->addr)); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 282 | |
Ben Hutchings | 58758aa | 2010-09-10 06:41:26 +0000 | [diff] [blame] | 283 | dma_free_coherent(&efx->pci_dev->dev, buffer->len, buffer->addr, |
| 284 | buffer->dma_addr); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 285 | buffer->addr = NULL; |
| 286 | buffer->entries = 0; |
| 287 | } |
| 288 | |
| 289 | /************************************************************************** |
| 290 | * |
| 291 | * Generic buffer handling |
| 292 | * These buffers are used for interrupt status and MAC stats |
| 293 | * |
| 294 | **************************************************************************/ |
| 295 | |
| 296 | int efx_nic_alloc_buffer(struct efx_nic *efx, struct efx_buffer *buffer, |
| 297 | unsigned int len) |
| 298 | { |
| 299 | buffer->addr = pci_alloc_consistent(efx->pci_dev, len, |
| 300 | &buffer->dma_addr); |
| 301 | if (!buffer->addr) |
| 302 | return -ENOMEM; |
| 303 | buffer->len = len; |
| 304 | memset(buffer->addr, 0, len); |
| 305 | return 0; |
| 306 | } |
| 307 | |
| 308 | void efx_nic_free_buffer(struct efx_nic *efx, struct efx_buffer *buffer) |
| 309 | { |
| 310 | if (buffer->addr) { |
| 311 | pci_free_consistent(efx->pci_dev, buffer->len, |
| 312 | buffer->addr, buffer->dma_addr); |
| 313 | buffer->addr = NULL; |
| 314 | } |
| 315 | } |
| 316 | |
| 317 | /************************************************************************** |
| 318 | * |
| 319 | * TX path |
| 320 | * |
| 321 | **************************************************************************/ |
| 322 | |
| 323 | /* Returns a pointer to the specified transmit descriptor in the TX |
| 324 | * descriptor queue belonging to the specified channel. |
| 325 | */ |
| 326 | static inline efx_qword_t * |
| 327 | efx_tx_desc(struct efx_tx_queue *tx_queue, unsigned int index) |
| 328 | { |
Eric Dumazet | 807540b | 2010-09-23 05:40:09 +0000 | [diff] [blame] | 329 | return ((efx_qword_t *) (tx_queue->txd.addr)) + index; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 330 | } |
| 331 | |
| 332 | /* This writes to the TX_DESC_WPTR; write pointer for TX descriptor ring */ |
| 333 | static inline void efx_notify_tx_desc(struct efx_tx_queue *tx_queue) |
| 334 | { |
| 335 | unsigned write_ptr; |
| 336 | efx_dword_t reg; |
| 337 | |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 338 | write_ptr = tx_queue->write_count & tx_queue->ptr_mask; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 339 | EFX_POPULATE_DWORD_1(reg, FRF_AZ_TX_DESC_WPTR_DWORD, write_ptr); |
| 340 | efx_writed_page(tx_queue->efx, ®, |
| 341 | FR_AZ_TX_DESC_UPD_DWORD_P0, tx_queue->queue); |
| 342 | } |
| 343 | |
Ben Hutchings | cd38557 | 2010-11-15 23:53:11 +0000 | [diff] [blame] | 344 | /* Write pointer and first descriptor for TX descriptor ring */ |
| 345 | static inline void efx_push_tx_desc(struct efx_tx_queue *tx_queue, |
| 346 | const efx_qword_t *txd) |
| 347 | { |
| 348 | unsigned write_ptr; |
| 349 | efx_oword_t reg; |
| 350 | |
| 351 | BUILD_BUG_ON(FRF_AZ_TX_DESC_LBN != 0); |
| 352 | BUILD_BUG_ON(FR_AA_TX_DESC_UPD_KER != FR_BZ_TX_DESC_UPD_P0); |
| 353 | |
| 354 | write_ptr = tx_queue->write_count & tx_queue->ptr_mask; |
| 355 | EFX_POPULATE_OWORD_2(reg, FRF_AZ_TX_DESC_PUSH_CMD, true, |
| 356 | FRF_AZ_TX_DESC_WPTR, write_ptr); |
| 357 | reg.qword[0] = *txd; |
| 358 | efx_writeo_page(tx_queue->efx, ®, |
| 359 | FR_BZ_TX_DESC_UPD_P0, tx_queue->queue); |
| 360 | } |
| 361 | |
| 362 | static inline bool |
| 363 | efx_may_push_tx_desc(struct efx_tx_queue *tx_queue, unsigned int write_count) |
| 364 | { |
| 365 | unsigned empty_read_count = ACCESS_ONCE(tx_queue->empty_read_count); |
| 366 | |
| 367 | if (empty_read_count == 0) |
| 368 | return false; |
| 369 | |
| 370 | tx_queue->empty_read_count = 0; |
| 371 | return ((empty_read_count ^ write_count) & ~EFX_EMPTY_COUNT_VALID) == 0; |
| 372 | } |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 373 | |
| 374 | /* For each entry inserted into the software descriptor ring, create a |
| 375 | * descriptor in the hardware TX descriptor ring (in host memory), and |
| 376 | * write a doorbell. |
| 377 | */ |
| 378 | void efx_nic_push_buffers(struct efx_tx_queue *tx_queue) |
| 379 | { |
| 380 | |
| 381 | struct efx_tx_buffer *buffer; |
| 382 | efx_qword_t *txd; |
| 383 | unsigned write_ptr; |
Ben Hutchings | cd38557 | 2010-11-15 23:53:11 +0000 | [diff] [blame] | 384 | unsigned old_write_count = tx_queue->write_count; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 385 | |
| 386 | BUG_ON(tx_queue->write_count == tx_queue->insert_count); |
| 387 | |
| 388 | do { |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 389 | write_ptr = tx_queue->write_count & tx_queue->ptr_mask; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 390 | buffer = &tx_queue->buffer[write_ptr]; |
| 391 | txd = efx_tx_desc(tx_queue, write_ptr); |
| 392 | ++tx_queue->write_count; |
| 393 | |
| 394 | /* Create TX descriptor ring entry */ |
| 395 | EFX_POPULATE_QWORD_4(*txd, |
| 396 | FSF_AZ_TX_KER_CONT, buffer->continuation, |
| 397 | FSF_AZ_TX_KER_BYTE_COUNT, buffer->len, |
| 398 | FSF_AZ_TX_KER_BUF_REGION, 0, |
| 399 | FSF_AZ_TX_KER_BUF_ADDR, buffer->dma_addr); |
| 400 | } while (tx_queue->write_count != tx_queue->insert_count); |
| 401 | |
| 402 | wmb(); /* Ensure descriptors are written before they are fetched */ |
Ben Hutchings | cd38557 | 2010-11-15 23:53:11 +0000 | [diff] [blame] | 403 | |
| 404 | if (efx_may_push_tx_desc(tx_queue, old_write_count)) { |
| 405 | txd = efx_tx_desc(tx_queue, |
| 406 | old_write_count & tx_queue->ptr_mask); |
| 407 | efx_push_tx_desc(tx_queue, txd); |
| 408 | ++tx_queue->pushes; |
| 409 | } else { |
| 410 | efx_notify_tx_desc(tx_queue); |
| 411 | } |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 412 | } |
| 413 | |
| 414 | /* Allocate hardware resources for a TX queue */ |
| 415 | int efx_nic_probe_tx(struct efx_tx_queue *tx_queue) |
| 416 | { |
| 417 | struct efx_nic *efx = tx_queue->efx; |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 418 | unsigned entries; |
| 419 | |
| 420 | entries = tx_queue->ptr_mask + 1; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 421 | return efx_alloc_special_buffer(efx, &tx_queue->txd, |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 422 | entries * sizeof(efx_qword_t)); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 423 | } |
| 424 | |
| 425 | void efx_nic_init_tx(struct efx_tx_queue *tx_queue) |
| 426 | { |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 427 | struct efx_nic *efx = tx_queue->efx; |
Ben Hutchings | 94b274b | 2011-01-10 21:18:20 +0000 | [diff] [blame] | 428 | efx_oword_t reg; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 429 | |
| 430 | tx_queue->flushed = FLUSH_NONE; |
| 431 | |
| 432 | /* Pin TX descriptor ring */ |
| 433 | efx_init_special_buffer(efx, &tx_queue->txd); |
| 434 | |
| 435 | /* Push TX descriptor ring to card */ |
Ben Hutchings | 94b274b | 2011-01-10 21:18:20 +0000 | [diff] [blame] | 436 | EFX_POPULATE_OWORD_10(reg, |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 437 | FRF_AZ_TX_DESCQ_EN, 1, |
| 438 | FRF_AZ_TX_ISCSI_DDIG_EN, 0, |
| 439 | FRF_AZ_TX_ISCSI_HDIG_EN, 0, |
| 440 | FRF_AZ_TX_DESCQ_BUF_BASE_ID, tx_queue->txd.index, |
| 441 | FRF_AZ_TX_DESCQ_EVQ_ID, |
| 442 | tx_queue->channel->channel, |
| 443 | FRF_AZ_TX_DESCQ_OWNER_ID, 0, |
| 444 | FRF_AZ_TX_DESCQ_LABEL, tx_queue->queue, |
| 445 | FRF_AZ_TX_DESCQ_SIZE, |
| 446 | __ffs(tx_queue->txd.entries), |
| 447 | FRF_AZ_TX_DESCQ_TYPE, 0, |
| 448 | FRF_BZ_TX_NON_IP_DROP_DIS, 1); |
| 449 | |
| 450 | if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) { |
Ben Hutchings | a4900ac | 2010-04-28 09:30:43 +0000 | [diff] [blame] | 451 | int csum = tx_queue->queue & EFX_TXQ_TYPE_OFFLOAD; |
Ben Hutchings | 94b274b | 2011-01-10 21:18:20 +0000 | [diff] [blame] | 452 | EFX_SET_OWORD_FIELD(reg, FRF_BZ_TX_IP_CHKSM_DIS, !csum); |
| 453 | EFX_SET_OWORD_FIELD(reg, FRF_BZ_TX_TCP_CHKSM_DIS, |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 454 | !csum); |
| 455 | } |
| 456 | |
Ben Hutchings | 94b274b | 2011-01-10 21:18:20 +0000 | [diff] [blame] | 457 | efx_writeo_table(efx, ®, efx->type->txd_ptr_tbl_base, |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 458 | tx_queue->queue); |
| 459 | |
| 460 | if (efx_nic_rev(efx) < EFX_REV_FALCON_B0) { |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 461 | /* Only 128 bits in this register */ |
Ben Hutchings | a4900ac | 2010-04-28 09:30:43 +0000 | [diff] [blame] | 462 | BUILD_BUG_ON(EFX_MAX_TX_QUEUES > 128); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 463 | |
| 464 | efx_reado(efx, ®, FR_AA_TX_CHKSM_CFG); |
Ben Hutchings | a4900ac | 2010-04-28 09:30:43 +0000 | [diff] [blame] | 465 | if (tx_queue->queue & EFX_TXQ_TYPE_OFFLOAD) |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 466 | clear_bit_le(tx_queue->queue, (void *)®); |
| 467 | else |
| 468 | set_bit_le(tx_queue->queue, (void *)®); |
| 469 | efx_writeo(efx, ®, FR_AA_TX_CHKSM_CFG); |
| 470 | } |
Ben Hutchings | 94b274b | 2011-01-10 21:18:20 +0000 | [diff] [blame] | 471 | |
| 472 | if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) { |
| 473 | EFX_POPULATE_OWORD_1(reg, |
| 474 | FRF_BZ_TX_PACE, |
| 475 | (tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI) ? |
| 476 | FFE_BZ_TX_PACE_OFF : |
| 477 | FFE_BZ_TX_PACE_RESERVED); |
| 478 | efx_writeo_table(efx, ®, FR_BZ_TX_PACE_TBL, |
| 479 | tx_queue->queue); |
| 480 | } |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 481 | } |
| 482 | |
| 483 | static void efx_flush_tx_queue(struct efx_tx_queue *tx_queue) |
| 484 | { |
| 485 | struct efx_nic *efx = tx_queue->efx; |
| 486 | efx_oword_t tx_flush_descq; |
| 487 | |
| 488 | tx_queue->flushed = FLUSH_PENDING; |
| 489 | |
| 490 | /* Post a flush command */ |
| 491 | EFX_POPULATE_OWORD_2(tx_flush_descq, |
| 492 | FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, |
| 493 | FRF_AZ_TX_FLUSH_DESCQ, tx_queue->queue); |
| 494 | efx_writeo(efx, &tx_flush_descq, FR_AZ_TX_FLUSH_DESCQ); |
| 495 | } |
| 496 | |
| 497 | void efx_nic_fini_tx(struct efx_tx_queue *tx_queue) |
| 498 | { |
| 499 | struct efx_nic *efx = tx_queue->efx; |
| 500 | efx_oword_t tx_desc_ptr; |
| 501 | |
| 502 | /* The queue should have been flushed */ |
| 503 | WARN_ON(tx_queue->flushed != FLUSH_DONE); |
| 504 | |
| 505 | /* Remove TX descriptor ring from card */ |
| 506 | EFX_ZERO_OWORD(tx_desc_ptr); |
| 507 | efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base, |
| 508 | tx_queue->queue); |
| 509 | |
| 510 | /* Unpin TX descriptor ring */ |
| 511 | efx_fini_special_buffer(efx, &tx_queue->txd); |
| 512 | } |
| 513 | |
| 514 | /* Free buffers backing TX queue */ |
| 515 | void efx_nic_remove_tx(struct efx_tx_queue *tx_queue) |
| 516 | { |
| 517 | efx_free_special_buffer(tx_queue->efx, &tx_queue->txd); |
| 518 | } |
| 519 | |
| 520 | /************************************************************************** |
| 521 | * |
| 522 | * RX path |
| 523 | * |
| 524 | **************************************************************************/ |
| 525 | |
| 526 | /* Returns a pointer to the specified descriptor in the RX descriptor queue */ |
| 527 | static inline efx_qword_t * |
| 528 | efx_rx_desc(struct efx_rx_queue *rx_queue, unsigned int index) |
| 529 | { |
Eric Dumazet | 807540b | 2010-09-23 05:40:09 +0000 | [diff] [blame] | 530 | return ((efx_qword_t *) (rx_queue->rxd.addr)) + index; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 531 | } |
| 532 | |
| 533 | /* This creates an entry in the RX descriptor queue */ |
| 534 | static inline void |
| 535 | efx_build_rx_desc(struct efx_rx_queue *rx_queue, unsigned index) |
| 536 | { |
| 537 | struct efx_rx_buffer *rx_buf; |
| 538 | efx_qword_t *rxd; |
| 539 | |
| 540 | rxd = efx_rx_desc(rx_queue, index); |
| 541 | rx_buf = efx_rx_buffer(rx_queue, index); |
| 542 | EFX_POPULATE_QWORD_3(*rxd, |
| 543 | FSF_AZ_RX_KER_BUF_SIZE, |
| 544 | rx_buf->len - |
| 545 | rx_queue->efx->type->rx_buffer_padding, |
| 546 | FSF_AZ_RX_KER_BUF_REGION, 0, |
| 547 | FSF_AZ_RX_KER_BUF_ADDR, rx_buf->dma_addr); |
| 548 | } |
| 549 | |
| 550 | /* This writes to the RX_DESC_WPTR register for the specified receive |
| 551 | * descriptor ring. |
| 552 | */ |
| 553 | void efx_nic_notify_rx_desc(struct efx_rx_queue *rx_queue) |
| 554 | { |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 555 | struct efx_nic *efx = rx_queue->efx; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 556 | efx_dword_t reg; |
| 557 | unsigned write_ptr; |
| 558 | |
| 559 | while (rx_queue->notified_count != rx_queue->added_count) { |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 560 | efx_build_rx_desc( |
| 561 | rx_queue, |
| 562 | rx_queue->notified_count & rx_queue->ptr_mask); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 563 | ++rx_queue->notified_count; |
| 564 | } |
| 565 | |
| 566 | wmb(); |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 567 | write_ptr = rx_queue->added_count & rx_queue->ptr_mask; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 568 | EFX_POPULATE_DWORD_1(reg, FRF_AZ_RX_DESC_WPTR_DWORD, write_ptr); |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 569 | efx_writed_page(efx, ®, FR_AZ_RX_DESC_UPD_DWORD_P0, |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 570 | efx_rx_queue_index(rx_queue)); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 571 | } |
| 572 | |
| 573 | int efx_nic_probe_rx(struct efx_rx_queue *rx_queue) |
| 574 | { |
| 575 | struct efx_nic *efx = rx_queue->efx; |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 576 | unsigned entries; |
| 577 | |
| 578 | entries = rx_queue->ptr_mask + 1; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 579 | return efx_alloc_special_buffer(efx, &rx_queue->rxd, |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 580 | entries * sizeof(efx_qword_t)); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 581 | } |
| 582 | |
| 583 | void efx_nic_init_rx(struct efx_rx_queue *rx_queue) |
| 584 | { |
| 585 | efx_oword_t rx_desc_ptr; |
| 586 | struct efx_nic *efx = rx_queue->efx; |
| 587 | bool is_b0 = efx_nic_rev(efx) >= EFX_REV_FALCON_B0; |
| 588 | bool iscsi_digest_en = is_b0; |
| 589 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 590 | netif_dbg(efx, hw, efx->net_dev, |
| 591 | "RX queue %d ring in special buffers %d-%d\n", |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 592 | efx_rx_queue_index(rx_queue), rx_queue->rxd.index, |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 593 | rx_queue->rxd.index + rx_queue->rxd.entries - 1); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 594 | |
| 595 | rx_queue->flushed = FLUSH_NONE; |
| 596 | |
| 597 | /* Pin RX descriptor ring */ |
| 598 | efx_init_special_buffer(efx, &rx_queue->rxd); |
| 599 | |
| 600 | /* Push RX descriptor ring to card */ |
| 601 | EFX_POPULATE_OWORD_10(rx_desc_ptr, |
| 602 | FRF_AZ_RX_ISCSI_DDIG_EN, iscsi_digest_en, |
| 603 | FRF_AZ_RX_ISCSI_HDIG_EN, iscsi_digest_en, |
| 604 | FRF_AZ_RX_DESCQ_BUF_BASE_ID, rx_queue->rxd.index, |
| 605 | FRF_AZ_RX_DESCQ_EVQ_ID, |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 606 | efx_rx_queue_channel(rx_queue)->channel, |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 607 | FRF_AZ_RX_DESCQ_OWNER_ID, 0, |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 608 | FRF_AZ_RX_DESCQ_LABEL, |
| 609 | efx_rx_queue_index(rx_queue), |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 610 | FRF_AZ_RX_DESCQ_SIZE, |
| 611 | __ffs(rx_queue->rxd.entries), |
| 612 | FRF_AZ_RX_DESCQ_TYPE, 0 /* kernel queue */ , |
| 613 | /* For >=B0 this is scatter so disable */ |
| 614 | FRF_AZ_RX_DESCQ_JUMBO, !is_b0, |
| 615 | FRF_AZ_RX_DESCQ_EN, 1); |
| 616 | efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base, |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 617 | efx_rx_queue_index(rx_queue)); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 618 | } |
| 619 | |
| 620 | static void efx_flush_rx_queue(struct efx_rx_queue *rx_queue) |
| 621 | { |
| 622 | struct efx_nic *efx = rx_queue->efx; |
| 623 | efx_oword_t rx_flush_descq; |
| 624 | |
| 625 | rx_queue->flushed = FLUSH_PENDING; |
| 626 | |
| 627 | /* Post a flush command */ |
| 628 | EFX_POPULATE_OWORD_2(rx_flush_descq, |
| 629 | FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 630 | FRF_AZ_RX_FLUSH_DESCQ, |
| 631 | efx_rx_queue_index(rx_queue)); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 632 | efx_writeo(efx, &rx_flush_descq, FR_AZ_RX_FLUSH_DESCQ); |
| 633 | } |
| 634 | |
| 635 | void efx_nic_fini_rx(struct efx_rx_queue *rx_queue) |
| 636 | { |
| 637 | efx_oword_t rx_desc_ptr; |
| 638 | struct efx_nic *efx = rx_queue->efx; |
| 639 | |
| 640 | /* The queue should already have been flushed */ |
| 641 | WARN_ON(rx_queue->flushed != FLUSH_DONE); |
| 642 | |
| 643 | /* Remove RX descriptor ring from card */ |
| 644 | EFX_ZERO_OWORD(rx_desc_ptr); |
| 645 | efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base, |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 646 | efx_rx_queue_index(rx_queue)); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 647 | |
| 648 | /* Unpin RX descriptor ring */ |
| 649 | efx_fini_special_buffer(efx, &rx_queue->rxd); |
| 650 | } |
| 651 | |
| 652 | /* Free buffers backing RX queue */ |
| 653 | void efx_nic_remove_rx(struct efx_rx_queue *rx_queue) |
| 654 | { |
| 655 | efx_free_special_buffer(rx_queue->efx, &rx_queue->rxd); |
| 656 | } |
| 657 | |
| 658 | /************************************************************************** |
| 659 | * |
| 660 | * Event queue processing |
| 661 | * Event queues are processed by per-channel tasklets. |
| 662 | * |
| 663 | **************************************************************************/ |
| 664 | |
| 665 | /* Update a channel's event queue's read pointer (RPTR) register |
| 666 | * |
| 667 | * This writes the EVQ_RPTR_REG register for the specified channel's |
| 668 | * event queue. |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 669 | */ |
| 670 | void efx_nic_eventq_read_ack(struct efx_channel *channel) |
| 671 | { |
| 672 | efx_dword_t reg; |
| 673 | struct efx_nic *efx = channel->efx; |
| 674 | |
Ben Hutchings | d4fabcc | 2011-04-04 14:22:11 +0100 | [diff] [blame] | 675 | EFX_POPULATE_DWORD_1(reg, FRF_AZ_EVQ_RPTR, |
| 676 | channel->eventq_read_ptr & channel->eventq_mask); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 677 | efx_writed_table(efx, ®, efx->type->evq_rptr_tbl_base, |
| 678 | channel->channel); |
| 679 | } |
| 680 | |
| 681 | /* Use HW to insert a SW defined event */ |
stephen hemminger | d215697 | 2010-10-18 05:27:31 +0000 | [diff] [blame] | 682 | static void efx_generate_event(struct efx_channel *channel, efx_qword_t *event) |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 683 | { |
| 684 | efx_oword_t drv_ev_reg; |
| 685 | |
| 686 | BUILD_BUG_ON(FRF_AZ_DRV_EV_DATA_LBN != 0 || |
| 687 | FRF_AZ_DRV_EV_DATA_WIDTH != 64); |
| 688 | drv_ev_reg.u32[0] = event->u32[0]; |
| 689 | drv_ev_reg.u32[1] = event->u32[1]; |
| 690 | drv_ev_reg.u32[2] = 0; |
| 691 | drv_ev_reg.u32[3] = 0; |
| 692 | EFX_SET_OWORD_FIELD(drv_ev_reg, FRF_AZ_DRV_EV_QID, channel->channel); |
| 693 | efx_writeo(channel->efx, &drv_ev_reg, FR_AZ_DRV_EV); |
| 694 | } |
| 695 | |
| 696 | /* Handle a transmit completion event |
| 697 | * |
| 698 | * The NIC batches TX completion events; the message we receive is of |
| 699 | * the form "complete all TX events up to this index". |
| 700 | */ |
Ben Hutchings | fa236e1 | 2010-04-28 09:29:42 +0000 | [diff] [blame] | 701 | static int |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 702 | efx_handle_tx_event(struct efx_channel *channel, efx_qword_t *event) |
| 703 | { |
| 704 | unsigned int tx_ev_desc_ptr; |
| 705 | unsigned int tx_ev_q_label; |
| 706 | struct efx_tx_queue *tx_queue; |
| 707 | struct efx_nic *efx = channel->efx; |
Ben Hutchings | fa236e1 | 2010-04-28 09:29:42 +0000 | [diff] [blame] | 708 | int tx_packets = 0; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 709 | |
| 710 | if (likely(EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_COMP))) { |
| 711 | /* Transmit completion */ |
| 712 | tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_DESC_PTR); |
| 713 | tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL); |
Ben Hutchings | f7d12cd | 2010-09-10 06:41:47 +0000 | [diff] [blame] | 714 | tx_queue = efx_channel_get_tx_queue( |
| 715 | channel, tx_ev_q_label % EFX_TXQ_TYPES); |
Ben Hutchings | fa236e1 | 2010-04-28 09:29:42 +0000 | [diff] [blame] | 716 | tx_packets = ((tx_ev_desc_ptr - tx_queue->read_count) & |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 717 | tx_queue->ptr_mask); |
Ben Hutchings | fa236e1 | 2010-04-28 09:29:42 +0000 | [diff] [blame] | 718 | channel->irq_mod_score += tx_packets; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 719 | efx_xmit_done(tx_queue, tx_ev_desc_ptr); |
| 720 | } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_WQ_FF_FULL)) { |
| 721 | /* Rewrite the FIFO write pointer */ |
| 722 | tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL); |
Ben Hutchings | f7d12cd | 2010-09-10 06:41:47 +0000 | [diff] [blame] | 723 | tx_queue = efx_channel_get_tx_queue( |
| 724 | channel, tx_ev_q_label % EFX_TXQ_TYPES); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 725 | |
Ben Hutchings | 73ba7b6 | 2012-01-09 19:47:08 +0000 | [diff] [blame] | 726 | netif_tx_lock(efx->net_dev); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 727 | efx_notify_tx_desc(tx_queue); |
Ben Hutchings | 73ba7b6 | 2012-01-09 19:47:08 +0000 | [diff] [blame] | 728 | netif_tx_unlock(efx->net_dev); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 729 | } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_PKT_ERR) && |
| 730 | EFX_WORKAROUND_10727(efx)) { |
| 731 | efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH); |
| 732 | } else { |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 733 | netif_err(efx, tx_err, efx->net_dev, |
| 734 | "channel %d unexpected TX event " |
| 735 | EFX_QWORD_FMT"\n", channel->channel, |
| 736 | EFX_QWORD_VAL(*event)); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 737 | } |
Ben Hutchings | fa236e1 | 2010-04-28 09:29:42 +0000 | [diff] [blame] | 738 | |
| 739 | return tx_packets; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 740 | } |
| 741 | |
| 742 | /* Detect errors included in the rx_evt_pkt_ok bit. */ |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 743 | static u16 efx_handle_rx_not_ok(struct efx_rx_queue *rx_queue, |
| 744 | const efx_qword_t *event) |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 745 | { |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 746 | struct efx_channel *channel = efx_rx_queue_channel(rx_queue); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 747 | struct efx_nic *efx = rx_queue->efx; |
| 748 | bool rx_ev_buf_owner_id_err, rx_ev_ip_hdr_chksum_err; |
| 749 | bool rx_ev_tcp_udp_chksum_err, rx_ev_eth_crc_err; |
| 750 | bool rx_ev_frm_trunc, rx_ev_drib_nib, rx_ev_tobe_disc; |
| 751 | bool rx_ev_other_err, rx_ev_pause_frm; |
| 752 | bool rx_ev_hdr_type, rx_ev_mcast_pkt; |
| 753 | unsigned rx_ev_pkt_type; |
| 754 | |
| 755 | rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE); |
| 756 | rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT); |
| 757 | rx_ev_tobe_disc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_TOBE_DISC); |
| 758 | rx_ev_pkt_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_TYPE); |
| 759 | rx_ev_buf_owner_id_err = EFX_QWORD_FIELD(*event, |
| 760 | FSF_AZ_RX_EV_BUF_OWNER_ID_ERR); |
| 761 | rx_ev_ip_hdr_chksum_err = EFX_QWORD_FIELD(*event, |
| 762 | FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR); |
| 763 | rx_ev_tcp_udp_chksum_err = EFX_QWORD_FIELD(*event, |
| 764 | FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR); |
| 765 | rx_ev_eth_crc_err = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_ETH_CRC_ERR); |
| 766 | rx_ev_frm_trunc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_FRM_TRUNC); |
| 767 | rx_ev_drib_nib = ((efx_nic_rev(efx) >= EFX_REV_FALCON_B0) ? |
| 768 | 0 : EFX_QWORD_FIELD(*event, FSF_AA_RX_EV_DRIB_NIB)); |
| 769 | rx_ev_pause_frm = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PAUSE_FRM_ERR); |
| 770 | |
| 771 | /* Every error apart from tobe_disc and pause_frm */ |
| 772 | rx_ev_other_err = (rx_ev_drib_nib | rx_ev_tcp_udp_chksum_err | |
| 773 | rx_ev_buf_owner_id_err | rx_ev_eth_crc_err | |
| 774 | rx_ev_frm_trunc | rx_ev_ip_hdr_chksum_err); |
| 775 | |
| 776 | /* Count errors that are not in MAC stats. Ignore expected |
| 777 | * checksum errors during self-test. */ |
| 778 | if (rx_ev_frm_trunc) |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 779 | ++channel->n_rx_frm_trunc; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 780 | else if (rx_ev_tobe_disc) |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 781 | ++channel->n_rx_tobe_disc; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 782 | else if (!efx->loopback_selftest) { |
| 783 | if (rx_ev_ip_hdr_chksum_err) |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 784 | ++channel->n_rx_ip_hdr_chksum_err; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 785 | else if (rx_ev_tcp_udp_chksum_err) |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 786 | ++channel->n_rx_tcp_udp_chksum_err; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 787 | } |
| 788 | |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 789 | /* TOBE_DISC is expected on unicast mismatches; don't print out an |
| 790 | * error message. FRM_TRUNC indicates RXDP dropped the packet due |
| 791 | * to a FIFO overflow. |
| 792 | */ |
Ben Hutchings | 5f3f9d6 | 2011-11-04 22:29:14 +0000 | [diff] [blame] | 793 | #ifdef DEBUG |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 794 | if (rx_ev_other_err && net_ratelimit()) { |
| 795 | netif_dbg(efx, rx_err, efx->net_dev, |
| 796 | " RX queue %d unexpected RX event " |
| 797 | EFX_QWORD_FMT "%s%s%s%s%s%s%s%s\n", |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 798 | efx_rx_queue_index(rx_queue), EFX_QWORD_VAL(*event), |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 799 | rx_ev_buf_owner_id_err ? " [OWNER_ID_ERR]" : "", |
| 800 | rx_ev_ip_hdr_chksum_err ? |
| 801 | " [IP_HDR_CHKSUM_ERR]" : "", |
| 802 | rx_ev_tcp_udp_chksum_err ? |
| 803 | " [TCP_UDP_CHKSUM_ERR]" : "", |
| 804 | rx_ev_eth_crc_err ? " [ETH_CRC_ERR]" : "", |
| 805 | rx_ev_frm_trunc ? " [FRM_TRUNC]" : "", |
| 806 | rx_ev_drib_nib ? " [DRIB_NIB]" : "", |
| 807 | rx_ev_tobe_disc ? " [TOBE_DISC]" : "", |
| 808 | rx_ev_pause_frm ? " [PAUSE]" : ""); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 809 | } |
| 810 | #endif |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 811 | |
| 812 | /* The frame must be discarded if any of these are true. */ |
| 813 | return (rx_ev_eth_crc_err | rx_ev_frm_trunc | rx_ev_drib_nib | |
| 814 | rx_ev_tobe_disc | rx_ev_pause_frm) ? |
| 815 | EFX_RX_PKT_DISCARD : 0; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 816 | } |
| 817 | |
| 818 | /* Handle receive events that are not in-order. */ |
| 819 | static void |
| 820 | efx_handle_rx_bad_index(struct efx_rx_queue *rx_queue, unsigned index) |
| 821 | { |
| 822 | struct efx_nic *efx = rx_queue->efx; |
| 823 | unsigned expected, dropped; |
| 824 | |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 825 | expected = rx_queue->removed_count & rx_queue->ptr_mask; |
| 826 | dropped = (index - expected) & rx_queue->ptr_mask; |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 827 | netif_info(efx, rx_err, efx->net_dev, |
| 828 | "dropped %d events (index=%d expected=%d)\n", |
| 829 | dropped, index, expected); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 830 | |
| 831 | efx_schedule_reset(efx, EFX_WORKAROUND_5676(efx) ? |
| 832 | RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE); |
| 833 | } |
| 834 | |
| 835 | /* Handle a packet received event |
| 836 | * |
| 837 | * The NIC gives a "discard" flag if it's a unicast packet with the |
| 838 | * wrong destination address |
| 839 | * Also "is multicast" and "matches multicast filter" flags can be used to |
| 840 | * discard non-matching multicast packets. |
| 841 | */ |
| 842 | static void |
| 843 | efx_handle_rx_event(struct efx_channel *channel, const efx_qword_t *event) |
| 844 | { |
| 845 | unsigned int rx_ev_desc_ptr, rx_ev_byte_cnt; |
| 846 | unsigned int rx_ev_hdr_type, rx_ev_mcast_pkt; |
| 847 | unsigned expected_ptr; |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 848 | bool rx_ev_pkt_ok; |
| 849 | u16 flags; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 850 | struct efx_rx_queue *rx_queue; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 851 | |
| 852 | /* Basic packet information */ |
| 853 | rx_ev_byte_cnt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_BYTE_CNT); |
| 854 | rx_ev_pkt_ok = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_OK); |
| 855 | rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE); |
| 856 | WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_JUMBO_CONT)); |
| 857 | WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_SOP) != 1); |
| 858 | WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_Q_LABEL) != |
| 859 | channel->channel); |
| 860 | |
Ben Hutchings | f7d12cd | 2010-09-10 06:41:47 +0000 | [diff] [blame] | 861 | rx_queue = efx_channel_get_rx_queue(channel); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 862 | |
| 863 | rx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_DESC_PTR); |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 864 | expected_ptr = rx_queue->removed_count & rx_queue->ptr_mask; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 865 | if (unlikely(rx_ev_desc_ptr != expected_ptr)) |
| 866 | efx_handle_rx_bad_index(rx_queue, rx_ev_desc_ptr); |
| 867 | |
| 868 | if (likely(rx_ev_pkt_ok)) { |
| 869 | /* If packet is marked as OK and packet type is TCP/IP or |
| 870 | * UDP/IP, then we can rely on the hardware checksum. |
| 871 | */ |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 872 | flags = (rx_ev_hdr_type == FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_TCP || |
| 873 | rx_ev_hdr_type == FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_UDP) ? |
| 874 | EFX_RX_PKT_CSUMMED : 0; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 875 | } else { |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 876 | flags = efx_handle_rx_not_ok(rx_queue, event); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 877 | } |
| 878 | |
| 879 | /* Detect multicast packets that didn't match the filter */ |
| 880 | rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT); |
| 881 | if (rx_ev_mcast_pkt) { |
| 882 | unsigned int rx_ev_mcast_hash_match = |
| 883 | EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_HASH_MATCH); |
| 884 | |
| 885 | if (unlikely(!rx_ev_mcast_hash_match)) { |
| 886 | ++channel->n_rx_mcast_mismatch; |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 887 | flags |= EFX_RX_PKT_DISCARD; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 888 | } |
| 889 | } |
| 890 | |
| 891 | channel->irq_mod_score += 2; |
| 892 | |
| 893 | /* Handle received packet */ |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 894 | efx_rx_packet(rx_queue, rx_ev_desc_ptr, rx_ev_byte_cnt, flags); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 895 | } |
| 896 | |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 897 | static void |
| 898 | efx_handle_generated_event(struct efx_channel *channel, efx_qword_t *event) |
| 899 | { |
| 900 | struct efx_nic *efx = channel->efx; |
| 901 | unsigned code; |
| 902 | |
| 903 | code = EFX_QWORD_FIELD(*event, FSF_AZ_DRV_GEN_EV_MAGIC); |
| 904 | if (code == EFX_CHANNEL_MAGIC_TEST(channel)) |
Ben Hutchings | d4fabcc | 2011-04-04 14:22:11 +0100 | [diff] [blame] | 905 | ; /* ignore */ |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 906 | else if (code == EFX_CHANNEL_MAGIC_FILL(channel)) |
| 907 | /* The queue must be empty, so we won't receive any rx |
| 908 | * events, so efx_process_channel() won't refill the |
| 909 | * queue. Refill it here */ |
Ben Hutchings | f7d12cd | 2010-09-10 06:41:47 +0000 | [diff] [blame] | 910 | efx_fast_push_rx_descriptors(efx_channel_get_rx_queue(channel)); |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 911 | else |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 912 | netif_dbg(efx, hw, efx->net_dev, "channel %d received " |
| 913 | "generated event "EFX_QWORD_FMT"\n", |
| 914 | channel->channel, EFX_QWORD_VAL(*event)); |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 915 | } |
| 916 | |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 917 | static void |
| 918 | efx_handle_driver_event(struct efx_channel *channel, efx_qword_t *event) |
| 919 | { |
| 920 | struct efx_nic *efx = channel->efx; |
| 921 | unsigned int ev_sub_code; |
| 922 | unsigned int ev_sub_data; |
| 923 | |
| 924 | ev_sub_code = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBCODE); |
| 925 | ev_sub_data = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA); |
| 926 | |
| 927 | switch (ev_sub_code) { |
| 928 | case FSE_AZ_TX_DESCQ_FLS_DONE_EV: |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 929 | netif_vdbg(efx, hw, efx->net_dev, "channel %d TXQ %d flushed\n", |
| 930 | channel->channel, ev_sub_data); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 931 | break; |
| 932 | case FSE_AZ_RX_DESCQ_FLS_DONE_EV: |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 933 | netif_vdbg(efx, hw, efx->net_dev, "channel %d RXQ %d flushed\n", |
| 934 | channel->channel, ev_sub_data); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 935 | break; |
| 936 | case FSE_AZ_EVQ_INIT_DONE_EV: |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 937 | netif_dbg(efx, hw, efx->net_dev, |
| 938 | "channel %d EVQ %d initialised\n", |
| 939 | channel->channel, ev_sub_data); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 940 | break; |
| 941 | case FSE_AZ_SRM_UPD_DONE_EV: |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 942 | netif_vdbg(efx, hw, efx->net_dev, |
| 943 | "channel %d SRAM update done\n", channel->channel); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 944 | break; |
| 945 | case FSE_AZ_WAKE_UP_EV: |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 946 | netif_vdbg(efx, hw, efx->net_dev, |
| 947 | "channel %d RXQ %d wakeup event\n", |
| 948 | channel->channel, ev_sub_data); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 949 | break; |
| 950 | case FSE_AZ_TIMER_EV: |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 951 | netif_vdbg(efx, hw, efx->net_dev, |
| 952 | "channel %d RX queue %d timer expired\n", |
| 953 | channel->channel, ev_sub_data); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 954 | break; |
| 955 | case FSE_AA_RX_RECOVER_EV: |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 956 | netif_err(efx, rx_err, efx->net_dev, |
| 957 | "channel %d seen DRIVER RX_RESET event. " |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 958 | "Resetting.\n", channel->channel); |
| 959 | atomic_inc(&efx->rx_reset); |
| 960 | efx_schedule_reset(efx, |
| 961 | EFX_WORKAROUND_6555(efx) ? |
| 962 | RESET_TYPE_RX_RECOVERY : |
| 963 | RESET_TYPE_DISABLE); |
| 964 | break; |
| 965 | case FSE_BZ_RX_DSC_ERROR_EV: |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 966 | netif_err(efx, rx_err, efx->net_dev, |
| 967 | "RX DMA Q %d reports descriptor fetch error." |
| 968 | " RX Q %d is disabled.\n", ev_sub_data, ev_sub_data); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 969 | efx_schedule_reset(efx, RESET_TYPE_RX_DESC_FETCH); |
| 970 | break; |
| 971 | case FSE_BZ_TX_DSC_ERROR_EV: |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 972 | netif_err(efx, tx_err, efx->net_dev, |
| 973 | "TX DMA Q %d reports descriptor fetch error." |
| 974 | " TX Q %d is disabled.\n", ev_sub_data, ev_sub_data); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 975 | efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH); |
| 976 | break; |
| 977 | default: |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 978 | netif_vdbg(efx, hw, efx->net_dev, |
| 979 | "channel %d unknown driver event code %d " |
| 980 | "data %04x\n", channel->channel, ev_sub_code, |
| 981 | ev_sub_data); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 982 | break; |
| 983 | } |
| 984 | } |
| 985 | |
Ben Hutchings | fa236e1 | 2010-04-28 09:29:42 +0000 | [diff] [blame] | 986 | int efx_nic_process_eventq(struct efx_channel *channel, int budget) |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 987 | { |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 988 | struct efx_nic *efx = channel->efx; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 989 | unsigned int read_ptr; |
| 990 | efx_qword_t event, *p_event; |
| 991 | int ev_code; |
Ben Hutchings | fa236e1 | 2010-04-28 09:29:42 +0000 | [diff] [blame] | 992 | int tx_packets = 0; |
| 993 | int spent = 0; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 994 | |
| 995 | read_ptr = channel->eventq_read_ptr; |
| 996 | |
Ben Hutchings | fa236e1 | 2010-04-28 09:29:42 +0000 | [diff] [blame] | 997 | for (;;) { |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 998 | p_event = efx_event(channel, read_ptr); |
| 999 | event = *p_event; |
| 1000 | |
| 1001 | if (!efx_event_present(&event)) |
| 1002 | /* End of events */ |
| 1003 | break; |
| 1004 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 1005 | netif_vdbg(channel->efx, intr, channel->efx->net_dev, |
| 1006 | "channel %d event is "EFX_QWORD_FMT"\n", |
| 1007 | channel->channel, EFX_QWORD_VAL(event)); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1008 | |
| 1009 | /* Clear this event by marking it all ones */ |
| 1010 | EFX_SET_QWORD(*p_event); |
| 1011 | |
Ben Hutchings | d4fabcc | 2011-04-04 14:22:11 +0100 | [diff] [blame] | 1012 | ++read_ptr; |
Ben Hutchings | fa236e1 | 2010-04-28 09:29:42 +0000 | [diff] [blame] | 1013 | |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1014 | ev_code = EFX_QWORD_FIELD(event, FSF_AZ_EV_CODE); |
| 1015 | |
| 1016 | switch (ev_code) { |
| 1017 | case FSE_AZ_EV_CODE_RX_EV: |
| 1018 | efx_handle_rx_event(channel, &event); |
Ben Hutchings | fa236e1 | 2010-04-28 09:29:42 +0000 | [diff] [blame] | 1019 | if (++spent == budget) |
| 1020 | goto out; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1021 | break; |
| 1022 | case FSE_AZ_EV_CODE_TX_EV: |
Ben Hutchings | fa236e1 | 2010-04-28 09:29:42 +0000 | [diff] [blame] | 1023 | tx_packets += efx_handle_tx_event(channel, &event); |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 1024 | if (tx_packets > efx->txq_entries) { |
Ben Hutchings | fa236e1 | 2010-04-28 09:29:42 +0000 | [diff] [blame] | 1025 | spent = budget; |
| 1026 | goto out; |
| 1027 | } |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1028 | break; |
| 1029 | case FSE_AZ_EV_CODE_DRV_GEN_EV: |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 1030 | efx_handle_generated_event(channel, &event); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1031 | break; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1032 | case FSE_AZ_EV_CODE_DRIVER_EV: |
| 1033 | efx_handle_driver_event(channel, &event); |
| 1034 | break; |
Ben Hutchings | 8880f4e | 2009-11-29 15:15:41 +0000 | [diff] [blame] | 1035 | case FSE_CZ_EV_CODE_MCDI_EV: |
| 1036 | efx_mcdi_process_event(channel, &event); |
| 1037 | break; |
Ben Hutchings | 40641ed | 2010-12-02 13:47:45 +0000 | [diff] [blame] | 1038 | case FSE_AZ_EV_CODE_GLOBAL_EV: |
| 1039 | if (efx->type->handle_global_event && |
| 1040 | efx->type->handle_global_event(channel, &event)) |
| 1041 | break; |
| 1042 | /* else fall through */ |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1043 | default: |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 1044 | netif_err(channel->efx, hw, channel->efx->net_dev, |
| 1045 | "channel %d unknown event type %d (data " |
| 1046 | EFX_QWORD_FMT ")\n", channel->channel, |
| 1047 | ev_code, EFX_QWORD_VAL(event)); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1048 | } |
Ben Hutchings | fa236e1 | 2010-04-28 09:29:42 +0000 | [diff] [blame] | 1049 | } |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1050 | |
Ben Hutchings | fa236e1 | 2010-04-28 09:29:42 +0000 | [diff] [blame] | 1051 | out: |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1052 | channel->eventq_read_ptr = read_ptr; |
Ben Hutchings | fa236e1 | 2010-04-28 09:29:42 +0000 | [diff] [blame] | 1053 | return spent; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1054 | } |
| 1055 | |
Ben Hutchings | d4fabcc | 2011-04-04 14:22:11 +0100 | [diff] [blame] | 1056 | /* Check whether an event is present in the eventq at the current |
| 1057 | * read pointer. Only useful for self-test. |
| 1058 | */ |
| 1059 | bool efx_nic_event_present(struct efx_channel *channel) |
| 1060 | { |
| 1061 | return efx_event_present(efx_event(channel, channel->eventq_read_ptr)); |
| 1062 | } |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1063 | |
| 1064 | /* Allocate buffer table entries for event queue */ |
| 1065 | int efx_nic_probe_eventq(struct efx_channel *channel) |
| 1066 | { |
| 1067 | struct efx_nic *efx = channel->efx; |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 1068 | unsigned entries; |
| 1069 | |
| 1070 | entries = channel->eventq_mask + 1; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1071 | return efx_alloc_special_buffer(efx, &channel->eventq, |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 1072 | entries * sizeof(efx_qword_t)); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1073 | } |
| 1074 | |
| 1075 | void efx_nic_init_eventq(struct efx_channel *channel) |
| 1076 | { |
Ben Hutchings | 8880f4e | 2009-11-29 15:15:41 +0000 | [diff] [blame] | 1077 | efx_oword_t reg; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1078 | struct efx_nic *efx = channel->efx; |
| 1079 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 1080 | netif_dbg(efx, hw, efx->net_dev, |
| 1081 | "channel %d event queue in special buffers %d-%d\n", |
| 1082 | channel->channel, channel->eventq.index, |
| 1083 | channel->eventq.index + channel->eventq.entries - 1); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1084 | |
Ben Hutchings | 8880f4e | 2009-11-29 15:15:41 +0000 | [diff] [blame] | 1085 | if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) { |
| 1086 | EFX_POPULATE_OWORD_3(reg, |
| 1087 | FRF_CZ_TIMER_Q_EN, 1, |
| 1088 | FRF_CZ_HOST_NOTIFY_MODE, 0, |
| 1089 | FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS); |
| 1090 | efx_writeo_table(efx, ®, FR_BZ_TIMER_TBL, channel->channel); |
| 1091 | } |
| 1092 | |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1093 | /* Pin event queue buffer */ |
| 1094 | efx_init_special_buffer(efx, &channel->eventq); |
| 1095 | |
| 1096 | /* Fill event queue with all ones (i.e. empty events) */ |
| 1097 | memset(channel->eventq.addr, 0xff, channel->eventq.len); |
| 1098 | |
| 1099 | /* Push event queue to card */ |
Ben Hutchings | 8880f4e | 2009-11-29 15:15:41 +0000 | [diff] [blame] | 1100 | EFX_POPULATE_OWORD_3(reg, |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1101 | FRF_AZ_EVQ_EN, 1, |
| 1102 | FRF_AZ_EVQ_SIZE, __ffs(channel->eventq.entries), |
| 1103 | FRF_AZ_EVQ_BUF_BASE_ID, channel->eventq.index); |
Ben Hutchings | 8880f4e | 2009-11-29 15:15:41 +0000 | [diff] [blame] | 1104 | efx_writeo_table(efx, ®, efx->type->evq_ptr_tbl_base, |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1105 | channel->channel); |
| 1106 | |
| 1107 | efx->type->push_irq_moderation(channel); |
| 1108 | } |
| 1109 | |
| 1110 | void efx_nic_fini_eventq(struct efx_channel *channel) |
| 1111 | { |
Ben Hutchings | 8880f4e | 2009-11-29 15:15:41 +0000 | [diff] [blame] | 1112 | efx_oword_t reg; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1113 | struct efx_nic *efx = channel->efx; |
| 1114 | |
| 1115 | /* Remove event queue from card */ |
Ben Hutchings | 8880f4e | 2009-11-29 15:15:41 +0000 | [diff] [blame] | 1116 | EFX_ZERO_OWORD(reg); |
| 1117 | efx_writeo_table(efx, ®, efx->type->evq_ptr_tbl_base, |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1118 | channel->channel); |
Ben Hutchings | 8880f4e | 2009-11-29 15:15:41 +0000 | [diff] [blame] | 1119 | if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) |
| 1120 | efx_writeo_table(efx, ®, FR_BZ_TIMER_TBL, channel->channel); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1121 | |
| 1122 | /* Unpin event queue */ |
| 1123 | efx_fini_special_buffer(efx, &channel->eventq); |
| 1124 | } |
| 1125 | |
| 1126 | /* Free buffers backing event queue */ |
| 1127 | void efx_nic_remove_eventq(struct efx_channel *channel) |
| 1128 | { |
| 1129 | efx_free_special_buffer(channel->efx, &channel->eventq); |
| 1130 | } |
| 1131 | |
| 1132 | |
Steve Hodgson | d730dc5 | 2010-06-01 11:19:09 +0000 | [diff] [blame] | 1133 | void efx_nic_generate_test_event(struct efx_channel *channel) |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1134 | { |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 1135 | unsigned int magic = EFX_CHANNEL_MAGIC_TEST(channel); |
| 1136 | efx_qword_t test_event; |
| 1137 | |
| 1138 | EFX_POPULATE_QWORD_2(test_event, FSF_AZ_EV_CODE, |
| 1139 | FSE_AZ_EV_CODE_DRV_GEN_EV, |
| 1140 | FSF_AZ_DRV_GEN_EV_MAGIC, magic); |
| 1141 | efx_generate_event(channel, &test_event); |
| 1142 | } |
| 1143 | |
| 1144 | void efx_nic_generate_fill_event(struct efx_channel *channel) |
| 1145 | { |
| 1146 | unsigned int magic = EFX_CHANNEL_MAGIC_FILL(channel); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1147 | efx_qword_t test_event; |
| 1148 | |
| 1149 | EFX_POPULATE_QWORD_2(test_event, FSF_AZ_EV_CODE, |
| 1150 | FSE_AZ_EV_CODE_DRV_GEN_EV, |
| 1151 | FSF_AZ_DRV_GEN_EV_MAGIC, magic); |
| 1152 | efx_generate_event(channel, &test_event); |
| 1153 | } |
| 1154 | |
| 1155 | /************************************************************************** |
| 1156 | * |
| 1157 | * Flush handling |
| 1158 | * |
| 1159 | **************************************************************************/ |
| 1160 | |
| 1161 | |
| 1162 | static void efx_poll_flush_events(struct efx_nic *efx) |
| 1163 | { |
Ben Hutchings | f7d12cd | 2010-09-10 06:41:47 +0000 | [diff] [blame] | 1164 | struct efx_channel *channel = efx_get_channel(efx, 0); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1165 | struct efx_tx_queue *tx_queue; |
| 1166 | struct efx_rx_queue *rx_queue; |
| 1167 | unsigned int read_ptr = channel->eventq_read_ptr; |
Ben Hutchings | d4fabcc | 2011-04-04 14:22:11 +0100 | [diff] [blame] | 1168 | unsigned int end_ptr = read_ptr + channel->eventq_mask - 1; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1169 | |
| 1170 | do { |
| 1171 | efx_qword_t *event = efx_event(channel, read_ptr); |
| 1172 | int ev_code, ev_sub_code, ev_queue; |
| 1173 | bool ev_failed; |
| 1174 | |
| 1175 | if (!efx_event_present(event)) |
| 1176 | break; |
| 1177 | |
| 1178 | ev_code = EFX_QWORD_FIELD(*event, FSF_AZ_EV_CODE); |
| 1179 | ev_sub_code = EFX_QWORD_FIELD(*event, |
| 1180 | FSF_AZ_DRIVER_EV_SUBCODE); |
| 1181 | if (ev_code == FSE_AZ_EV_CODE_DRIVER_EV && |
| 1182 | ev_sub_code == FSE_AZ_TX_DESCQ_FLS_DONE_EV) { |
| 1183 | ev_queue = EFX_QWORD_FIELD(*event, |
| 1184 | FSF_AZ_DRIVER_EV_SUBDATA); |
Ben Hutchings | a4900ac | 2010-04-28 09:30:43 +0000 | [diff] [blame] | 1185 | if (ev_queue < EFX_TXQ_TYPES * efx->n_tx_channels) { |
Ben Hutchings | f7d12cd | 2010-09-10 06:41:47 +0000 | [diff] [blame] | 1186 | tx_queue = efx_get_tx_queue( |
| 1187 | efx, ev_queue / EFX_TXQ_TYPES, |
| 1188 | ev_queue % EFX_TXQ_TYPES); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1189 | tx_queue->flushed = FLUSH_DONE; |
| 1190 | } |
| 1191 | } else if (ev_code == FSE_AZ_EV_CODE_DRIVER_EV && |
| 1192 | ev_sub_code == FSE_AZ_RX_DESCQ_FLS_DONE_EV) { |
| 1193 | ev_queue = EFX_QWORD_FIELD( |
| 1194 | *event, FSF_AZ_DRIVER_EV_RX_DESCQ_ID); |
| 1195 | ev_failed = EFX_QWORD_FIELD( |
| 1196 | *event, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL); |
Ben Hutchings | a4900ac | 2010-04-28 09:30:43 +0000 | [diff] [blame] | 1197 | if (ev_queue < efx->n_rx_channels) { |
Ben Hutchings | f7d12cd | 2010-09-10 06:41:47 +0000 | [diff] [blame] | 1198 | rx_queue = efx_get_rx_queue(efx, ev_queue); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1199 | rx_queue->flushed = |
| 1200 | ev_failed ? FLUSH_FAILED : FLUSH_DONE; |
| 1201 | } |
| 1202 | } |
| 1203 | |
| 1204 | /* We're about to destroy the queue anyway, so |
| 1205 | * it's ok to throw away every non-flush event */ |
| 1206 | EFX_SET_QWORD(*event); |
| 1207 | |
Ben Hutchings | d4fabcc | 2011-04-04 14:22:11 +0100 | [diff] [blame] | 1208 | ++read_ptr; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1209 | } while (read_ptr != end_ptr); |
| 1210 | |
| 1211 | channel->eventq_read_ptr = read_ptr; |
| 1212 | } |
| 1213 | |
| 1214 | /* Handle tx and rx flushes at the same time, since they run in |
| 1215 | * parallel in the hardware and there's no reason for us to |
| 1216 | * serialise them */ |
| 1217 | int efx_nic_flush_queues(struct efx_nic *efx) |
| 1218 | { |
Ben Hutchings | f7d12cd | 2010-09-10 06:41:47 +0000 | [diff] [blame] | 1219 | struct efx_channel *channel; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1220 | struct efx_rx_queue *rx_queue; |
| 1221 | struct efx_tx_queue *tx_queue; |
| 1222 | int i, tx_pending, rx_pending; |
| 1223 | |
| 1224 | /* If necessary prepare the hardware for flushing */ |
| 1225 | efx->type->prepare_flush(efx); |
| 1226 | |
| 1227 | /* Flush all tx queues in parallel */ |
Ben Hutchings | f7d12cd | 2010-09-10 06:41:47 +0000 | [diff] [blame] | 1228 | efx_for_each_channel(channel, efx) { |
Ben Hutchings | 94b274b | 2011-01-10 21:18:20 +0000 | [diff] [blame] | 1229 | efx_for_each_possible_channel_tx_queue(tx_queue, channel) { |
| 1230 | if (tx_queue->initialised) |
| 1231 | efx_flush_tx_queue(tx_queue); |
| 1232 | } |
Ben Hutchings | f7d12cd | 2010-09-10 06:41:47 +0000 | [diff] [blame] | 1233 | } |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1234 | |
| 1235 | /* The hardware supports four concurrent rx flushes, each of which may |
| 1236 | * need to be retried if there is an outstanding descriptor fetch */ |
| 1237 | for (i = 0; i < EFX_FLUSH_POLL_COUNT; ++i) { |
| 1238 | rx_pending = tx_pending = 0; |
Ben Hutchings | f7d12cd | 2010-09-10 06:41:47 +0000 | [diff] [blame] | 1239 | efx_for_each_channel(channel, efx) { |
| 1240 | efx_for_each_channel_rx_queue(rx_queue, channel) { |
| 1241 | if (rx_queue->flushed == FLUSH_PENDING) |
| 1242 | ++rx_pending; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1243 | } |
| 1244 | } |
Ben Hutchings | f7d12cd | 2010-09-10 06:41:47 +0000 | [diff] [blame] | 1245 | efx_for_each_channel(channel, efx) { |
| 1246 | efx_for_each_channel_rx_queue(rx_queue, channel) { |
| 1247 | if (rx_pending == EFX_RX_FLUSH_COUNT) |
| 1248 | break; |
| 1249 | if (rx_queue->flushed == FLUSH_FAILED || |
| 1250 | rx_queue->flushed == FLUSH_NONE) { |
| 1251 | efx_flush_rx_queue(rx_queue); |
| 1252 | ++rx_pending; |
| 1253 | } |
| 1254 | } |
Ben Hutchings | 94b274b | 2011-01-10 21:18:20 +0000 | [diff] [blame] | 1255 | efx_for_each_possible_channel_tx_queue(tx_queue, channel) { |
| 1256 | if (tx_queue->initialised && |
| 1257 | tx_queue->flushed != FLUSH_DONE) |
Ben Hutchings | f7d12cd | 2010-09-10 06:41:47 +0000 | [diff] [blame] | 1258 | ++tx_pending; |
| 1259 | } |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1260 | } |
| 1261 | |
| 1262 | if (rx_pending == 0 && tx_pending == 0) |
| 1263 | return 0; |
| 1264 | |
| 1265 | msleep(EFX_FLUSH_INTERVAL); |
| 1266 | efx_poll_flush_events(efx); |
| 1267 | } |
| 1268 | |
| 1269 | /* Mark the queues as all flushed. We're going to return failure |
| 1270 | * leading to a reset, or fake up success anyway */ |
Ben Hutchings | f7d12cd | 2010-09-10 06:41:47 +0000 | [diff] [blame] | 1271 | efx_for_each_channel(channel, efx) { |
Ben Hutchings | 94b274b | 2011-01-10 21:18:20 +0000 | [diff] [blame] | 1272 | efx_for_each_possible_channel_tx_queue(tx_queue, channel) { |
| 1273 | if (tx_queue->initialised && |
| 1274 | tx_queue->flushed != FLUSH_DONE) |
Ben Hutchings | f7d12cd | 2010-09-10 06:41:47 +0000 | [diff] [blame] | 1275 | netif_err(efx, hw, efx->net_dev, |
| 1276 | "tx queue %d flush command timed out\n", |
| 1277 | tx_queue->queue); |
| 1278 | tx_queue->flushed = FLUSH_DONE; |
| 1279 | } |
| 1280 | efx_for_each_channel_rx_queue(rx_queue, channel) { |
| 1281 | if (rx_queue->flushed != FLUSH_DONE) |
| 1282 | netif_err(efx, hw, efx->net_dev, |
| 1283 | "rx queue %d flush command timed out\n", |
| 1284 | efx_rx_queue_index(rx_queue)); |
| 1285 | rx_queue->flushed = FLUSH_DONE; |
| 1286 | } |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1287 | } |
| 1288 | |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1289 | return -ETIMEDOUT; |
| 1290 | } |
| 1291 | |
| 1292 | /************************************************************************** |
| 1293 | * |
| 1294 | * Hardware interrupts |
| 1295 | * The hardware interrupt handler does very little work; all the event |
| 1296 | * queue processing is carried out by per-channel tasklets. |
| 1297 | * |
| 1298 | **************************************************************************/ |
| 1299 | |
| 1300 | /* Enable/disable/generate interrupts */ |
| 1301 | static inline void efx_nic_interrupts(struct efx_nic *efx, |
| 1302 | bool enabled, bool force) |
| 1303 | { |
| 1304 | efx_oword_t int_en_reg_ker; |
Ben Hutchings | 8880f4e | 2009-11-29 15:15:41 +0000 | [diff] [blame] | 1305 | |
| 1306 | EFX_POPULATE_OWORD_3(int_en_reg_ker, |
Ben Hutchings | 1646a6f | 2012-01-05 20:14:10 +0000 | [diff] [blame] | 1307 | FRF_AZ_KER_INT_LEVE_SEL, efx->irq_level, |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1308 | FRF_AZ_KER_INT_KER, force, |
| 1309 | FRF_AZ_DRV_INT_EN_KER, enabled); |
| 1310 | efx_writeo(efx, &int_en_reg_ker, FR_AZ_INT_EN_KER); |
| 1311 | } |
| 1312 | |
| 1313 | void efx_nic_enable_interrupts(struct efx_nic *efx) |
| 1314 | { |
| 1315 | struct efx_channel *channel; |
| 1316 | |
| 1317 | EFX_ZERO_OWORD(*((efx_oword_t *) efx->irq_status.addr)); |
| 1318 | wmb(); /* Ensure interrupt vector is clear before interrupts enabled */ |
| 1319 | |
| 1320 | /* Enable interrupts */ |
| 1321 | efx_nic_interrupts(efx, true, false); |
| 1322 | |
| 1323 | /* Force processing of all the channels to get the EVQ RPTRs up to |
| 1324 | date */ |
| 1325 | efx_for_each_channel(channel, efx) |
| 1326 | efx_schedule_channel(channel); |
| 1327 | } |
| 1328 | |
| 1329 | void efx_nic_disable_interrupts(struct efx_nic *efx) |
| 1330 | { |
| 1331 | /* Disable interrupts */ |
| 1332 | efx_nic_interrupts(efx, false, false); |
| 1333 | } |
| 1334 | |
| 1335 | /* Generate a test interrupt |
| 1336 | * Interrupt must already have been enabled, otherwise nasty things |
| 1337 | * may happen. |
| 1338 | */ |
| 1339 | void efx_nic_generate_interrupt(struct efx_nic *efx) |
| 1340 | { |
| 1341 | efx_nic_interrupts(efx, true, true); |
| 1342 | } |
| 1343 | |
| 1344 | /* Process a fatal interrupt |
| 1345 | * Disable bus mastering ASAP and schedule a reset |
| 1346 | */ |
| 1347 | irqreturn_t efx_nic_fatal_interrupt(struct efx_nic *efx) |
| 1348 | { |
| 1349 | struct falcon_nic_data *nic_data = efx->nic_data; |
| 1350 | efx_oword_t *int_ker = efx->irq_status.addr; |
| 1351 | efx_oword_t fatal_intr; |
| 1352 | int error, mem_perr; |
| 1353 | |
| 1354 | efx_reado(efx, &fatal_intr, FR_AZ_FATAL_INTR_KER); |
| 1355 | error = EFX_OWORD_FIELD(fatal_intr, FRF_AZ_FATAL_INTR); |
| 1356 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 1357 | netif_err(efx, hw, efx->net_dev, "SYSTEM ERROR "EFX_OWORD_FMT" status " |
| 1358 | EFX_OWORD_FMT ": %s\n", EFX_OWORD_VAL(*int_ker), |
| 1359 | EFX_OWORD_VAL(fatal_intr), |
| 1360 | error ? "disabling bus mastering" : "no recognised error"); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1361 | |
| 1362 | /* If this is a memory parity error dump which blocks are offending */ |
Steve Hodgson | 97e1eaa | 2010-04-28 09:28:52 +0000 | [diff] [blame] | 1363 | mem_perr = (EFX_OWORD_FIELD(fatal_intr, FRF_AZ_MEM_PERR_INT_KER) || |
| 1364 | EFX_OWORD_FIELD(fatal_intr, FRF_AZ_SRM_PERR_INT_KER)); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1365 | if (mem_perr) { |
| 1366 | efx_oword_t reg; |
| 1367 | efx_reado(efx, ®, FR_AZ_MEM_STAT); |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 1368 | netif_err(efx, hw, efx->net_dev, |
| 1369 | "SYSTEM ERROR: memory parity error "EFX_OWORD_FMT"\n", |
| 1370 | EFX_OWORD_VAL(reg)); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1371 | } |
| 1372 | |
| 1373 | /* Disable both devices */ |
| 1374 | pci_clear_master(efx->pci_dev); |
| 1375 | if (efx_nic_is_dual_func(efx)) |
| 1376 | pci_clear_master(nic_data->pci_dev2); |
| 1377 | efx_nic_disable_interrupts(efx); |
| 1378 | |
| 1379 | /* Count errors and reset or disable the NIC accordingly */ |
| 1380 | if (efx->int_error_count == 0 || |
| 1381 | time_after(jiffies, efx->int_error_expire)) { |
| 1382 | efx->int_error_count = 0; |
| 1383 | efx->int_error_expire = |
| 1384 | jiffies + EFX_INT_ERROR_EXPIRE * HZ; |
| 1385 | } |
| 1386 | if (++efx->int_error_count < EFX_MAX_INT_ERRORS) { |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 1387 | netif_err(efx, hw, efx->net_dev, |
| 1388 | "SYSTEM ERROR - reset scheduled\n"); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1389 | efx_schedule_reset(efx, RESET_TYPE_INT_ERROR); |
| 1390 | } else { |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 1391 | netif_err(efx, hw, efx->net_dev, |
| 1392 | "SYSTEM ERROR - max number of errors seen." |
| 1393 | "NIC will be disabled\n"); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1394 | efx_schedule_reset(efx, RESET_TYPE_DISABLE); |
| 1395 | } |
Steve Hodgson | 6369545 | 2010-04-28 09:27:36 +0000 | [diff] [blame] | 1396 | |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1397 | return IRQ_HANDLED; |
| 1398 | } |
| 1399 | |
| 1400 | /* Handle a legacy interrupt |
| 1401 | * Acknowledges the interrupt and schedule event queue processing. |
| 1402 | */ |
| 1403 | static irqreturn_t efx_legacy_interrupt(int irq, void *dev_id) |
| 1404 | { |
| 1405 | struct efx_nic *efx = dev_id; |
| 1406 | efx_oword_t *int_ker = efx->irq_status.addr; |
| 1407 | irqreturn_t result = IRQ_NONE; |
| 1408 | struct efx_channel *channel; |
| 1409 | efx_dword_t reg; |
| 1410 | u32 queues; |
| 1411 | int syserr; |
| 1412 | |
Ben Hutchings | 94dec6a | 2010-12-07 19:24:45 +0000 | [diff] [blame] | 1413 | /* Could this be ours? If interrupts are disabled then the |
| 1414 | * channel state may not be valid. |
| 1415 | */ |
| 1416 | if (!efx->legacy_irq_enabled) |
| 1417 | return result; |
| 1418 | |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1419 | /* Read the ISR which also ACKs the interrupts */ |
| 1420 | efx_readd(efx, ®, FR_BZ_INT_ISR0); |
| 1421 | queues = EFX_EXTRACT_DWORD(reg, 0, 31); |
| 1422 | |
Ben Hutchings | 1646a6f | 2012-01-05 20:14:10 +0000 | [diff] [blame] | 1423 | /* Handle non-event-queue sources */ |
| 1424 | if (queues & (1U << efx->irq_level)) { |
Steve Hodgson | 6369545 | 2010-04-28 09:27:36 +0000 | [diff] [blame] | 1425 | syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT); |
| 1426 | if (unlikely(syserr)) |
| 1427 | return efx_nic_fatal_interrupt(efx); |
Ben Hutchings | 1646a6f | 2012-01-05 20:14:10 +0000 | [diff] [blame] | 1428 | efx->last_irq_cpu = raw_smp_processor_id(); |
Steve Hodgson | 6369545 | 2010-04-28 09:27:36 +0000 | [diff] [blame] | 1429 | } |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1430 | |
Ben Hutchings | 8880f4e | 2009-11-29 15:15:41 +0000 | [diff] [blame] | 1431 | if (queues != 0) { |
| 1432 | if (EFX_WORKAROUND_15783(efx)) |
| 1433 | efx->irq_zero_count = 0; |
| 1434 | |
| 1435 | /* Schedule processing of any interrupting queues */ |
| 1436 | efx_for_each_channel(channel, efx) { |
| 1437 | if (queues & 1) |
Ben Hutchings | 1646a6f | 2012-01-05 20:14:10 +0000 | [diff] [blame] | 1438 | efx_schedule_channel_irq(channel); |
Ben Hutchings | 8880f4e | 2009-11-29 15:15:41 +0000 | [diff] [blame] | 1439 | queues >>= 1; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1440 | } |
Ben Hutchings | 8880f4e | 2009-11-29 15:15:41 +0000 | [diff] [blame] | 1441 | result = IRQ_HANDLED; |
| 1442 | |
Steve Hodgson | 41b7e4c | 2010-04-28 09:28:27 +0000 | [diff] [blame] | 1443 | } else if (EFX_WORKAROUND_15783(efx)) { |
Ben Hutchings | 8880f4e | 2009-11-29 15:15:41 +0000 | [diff] [blame] | 1444 | efx_qword_t *event; |
| 1445 | |
Steve Hodgson | 41b7e4c | 2010-04-28 09:28:27 +0000 | [diff] [blame] | 1446 | /* We can't return IRQ_HANDLED more than once on seeing ISR=0 |
| 1447 | * because this might be a shared interrupt. */ |
| 1448 | if (efx->irq_zero_count++ == 0) |
| 1449 | result = IRQ_HANDLED; |
| 1450 | |
| 1451 | /* Ensure we schedule or rearm all event queues */ |
Ben Hutchings | 8880f4e | 2009-11-29 15:15:41 +0000 | [diff] [blame] | 1452 | efx_for_each_channel(channel, efx) { |
| 1453 | event = efx_event(channel, channel->eventq_read_ptr); |
| 1454 | if (efx_event_present(event)) |
Ben Hutchings | 1646a6f | 2012-01-05 20:14:10 +0000 | [diff] [blame] | 1455 | efx_schedule_channel_irq(channel); |
Steve Hodgson | 41b7e4c | 2010-04-28 09:28:27 +0000 | [diff] [blame] | 1456 | else |
| 1457 | efx_nic_eventq_read_ack(channel); |
Ben Hutchings | 8880f4e | 2009-11-29 15:15:41 +0000 | [diff] [blame] | 1458 | } |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1459 | } |
| 1460 | |
Ben Hutchings | 1646a6f | 2012-01-05 20:14:10 +0000 | [diff] [blame] | 1461 | if (result == IRQ_HANDLED) |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 1462 | netif_vdbg(efx, intr, efx->net_dev, |
| 1463 | "IRQ %d on CPU %d status " EFX_DWORD_FMT "\n", |
| 1464 | irq, raw_smp_processor_id(), EFX_DWORD_VAL(reg)); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1465 | |
| 1466 | return result; |
| 1467 | } |
| 1468 | |
| 1469 | /* Handle an MSI interrupt |
| 1470 | * |
| 1471 | * Handle an MSI hardware interrupt. This routine schedules event |
| 1472 | * queue processing. No interrupt acknowledgement cycle is necessary. |
| 1473 | * Also, we never need to check that the interrupt is for us, since |
| 1474 | * MSI interrupts cannot be shared. |
| 1475 | */ |
| 1476 | static irqreturn_t efx_msi_interrupt(int irq, void *dev_id) |
| 1477 | { |
Ben Hutchings | 4642610 | 2010-09-10 06:42:33 +0000 | [diff] [blame] | 1478 | struct efx_channel *channel = *(struct efx_channel **)dev_id; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1479 | struct efx_nic *efx = channel->efx; |
| 1480 | efx_oword_t *int_ker = efx->irq_status.addr; |
| 1481 | int syserr; |
| 1482 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 1483 | netif_vdbg(efx, intr, efx->net_dev, |
| 1484 | "IRQ %d on CPU %d status " EFX_OWORD_FMT "\n", |
| 1485 | irq, raw_smp_processor_id(), EFX_OWORD_VAL(*int_ker)); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1486 | |
Ben Hutchings | 1646a6f | 2012-01-05 20:14:10 +0000 | [diff] [blame] | 1487 | /* Handle non-event-queue sources */ |
| 1488 | if (channel->channel == efx->irq_level) { |
Steve Hodgson | 6369545 | 2010-04-28 09:27:36 +0000 | [diff] [blame] | 1489 | syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT); |
| 1490 | if (unlikely(syserr)) |
| 1491 | return efx_nic_fatal_interrupt(efx); |
Ben Hutchings | 1646a6f | 2012-01-05 20:14:10 +0000 | [diff] [blame] | 1492 | efx->last_irq_cpu = raw_smp_processor_id(); |
Steve Hodgson | 6369545 | 2010-04-28 09:27:36 +0000 | [diff] [blame] | 1493 | } |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1494 | |
| 1495 | /* Schedule processing of the channel */ |
Ben Hutchings | 1646a6f | 2012-01-05 20:14:10 +0000 | [diff] [blame] | 1496 | efx_schedule_channel_irq(channel); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1497 | |
| 1498 | return IRQ_HANDLED; |
| 1499 | } |
| 1500 | |
| 1501 | |
| 1502 | /* Setup RSS indirection table. |
| 1503 | * This maps from the hash value of the packet to RXQ |
| 1504 | */ |
Ben Hutchings | 765c9f4 | 2010-06-30 05:06:28 +0000 | [diff] [blame] | 1505 | void efx_nic_push_rx_indir_table(struct efx_nic *efx) |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1506 | { |
Ben Hutchings | 765c9f4 | 2010-06-30 05:06:28 +0000 | [diff] [blame] | 1507 | size_t i = 0; |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1508 | efx_dword_t dword; |
| 1509 | |
| 1510 | if (efx_nic_rev(efx) < EFX_REV_FALCON_B0) |
| 1511 | return; |
| 1512 | |
Ben Hutchings | 765c9f4 | 2010-06-30 05:06:28 +0000 | [diff] [blame] | 1513 | BUILD_BUG_ON(ARRAY_SIZE(efx->rx_indir_table) != |
| 1514 | FR_BZ_RX_INDIRECTION_TBL_ROWS); |
| 1515 | |
| 1516 | for (i = 0; i < FR_BZ_RX_INDIRECTION_TBL_ROWS; i++) { |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1517 | EFX_POPULATE_DWORD_1(dword, FRF_BZ_IT_QUEUE, |
Ben Hutchings | 765c9f4 | 2010-06-30 05:06:28 +0000 | [diff] [blame] | 1518 | efx->rx_indir_table[i]); |
| 1519 | efx_writed_table(efx, &dword, FR_BZ_RX_INDIRECTION_TBL, i); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1520 | } |
| 1521 | } |
| 1522 | |
| 1523 | /* Hook interrupt handler(s) |
| 1524 | * Try MSI and then legacy interrupts. |
| 1525 | */ |
| 1526 | int efx_nic_init_interrupt(struct efx_nic *efx) |
| 1527 | { |
| 1528 | struct efx_channel *channel; |
| 1529 | int rc; |
| 1530 | |
| 1531 | if (!EFX_INT_MODE_USE_MSI(efx)) { |
| 1532 | irq_handler_t handler; |
| 1533 | if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) |
| 1534 | handler = efx_legacy_interrupt; |
| 1535 | else |
| 1536 | handler = falcon_legacy_interrupt_a1; |
| 1537 | |
| 1538 | rc = request_irq(efx->legacy_irq, handler, IRQF_SHARED, |
| 1539 | efx->name, efx); |
| 1540 | if (rc) { |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 1541 | netif_err(efx, drv, efx->net_dev, |
| 1542 | "failed to hook legacy IRQ %d\n", |
| 1543 | efx->pci_dev->irq); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1544 | goto fail1; |
| 1545 | } |
| 1546 | return 0; |
| 1547 | } |
| 1548 | |
| 1549 | /* Hook MSI or MSI-X interrupt */ |
| 1550 | efx_for_each_channel(channel, efx) { |
| 1551 | rc = request_irq(channel->irq, efx_msi_interrupt, |
| 1552 | IRQF_PROBE_SHARED, /* Not shared */ |
Ben Hutchings | 4642610 | 2010-09-10 06:42:33 +0000 | [diff] [blame] | 1553 | efx->channel_name[channel->channel], |
| 1554 | &efx->channel[channel->channel]); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1555 | if (rc) { |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 1556 | netif_err(efx, drv, efx->net_dev, |
| 1557 | "failed to hook IRQ %d\n", channel->irq); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1558 | goto fail2; |
| 1559 | } |
| 1560 | } |
| 1561 | |
| 1562 | return 0; |
| 1563 | |
| 1564 | fail2: |
| 1565 | efx_for_each_channel(channel, efx) |
Ben Hutchings | 4642610 | 2010-09-10 06:42:33 +0000 | [diff] [blame] | 1566 | free_irq(channel->irq, &efx->channel[channel->channel]); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1567 | fail1: |
| 1568 | return rc; |
| 1569 | } |
| 1570 | |
| 1571 | void efx_nic_fini_interrupt(struct efx_nic *efx) |
| 1572 | { |
| 1573 | struct efx_channel *channel; |
| 1574 | efx_oword_t reg; |
| 1575 | |
| 1576 | /* Disable MSI/MSI-X interrupts */ |
| 1577 | efx_for_each_channel(channel, efx) { |
| 1578 | if (channel->irq) |
Ben Hutchings | 4642610 | 2010-09-10 06:42:33 +0000 | [diff] [blame] | 1579 | free_irq(channel->irq, &efx->channel[channel->channel]); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1580 | } |
| 1581 | |
| 1582 | /* ACK legacy interrupt */ |
| 1583 | if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) |
| 1584 | efx_reado(efx, ®, FR_BZ_INT_ISR0); |
| 1585 | else |
| 1586 | falcon_irq_ack_a1(efx); |
| 1587 | |
| 1588 | /* Disable legacy interrupt */ |
| 1589 | if (efx->legacy_irq) |
| 1590 | free_irq(efx->legacy_irq, efx); |
| 1591 | } |
| 1592 | |
| 1593 | u32 efx_nic_fpga_ver(struct efx_nic *efx) |
| 1594 | { |
| 1595 | efx_oword_t altera_build; |
| 1596 | efx_reado(efx, &altera_build, FR_AZ_ALTERA_BUILD); |
| 1597 | return EFX_OWORD_FIELD(altera_build, FRF_AZ_ALTERA_BUILD_VER); |
| 1598 | } |
| 1599 | |
| 1600 | void efx_nic_init_common(struct efx_nic *efx) |
| 1601 | { |
| 1602 | efx_oword_t temp; |
| 1603 | |
| 1604 | /* Set positions of descriptor caches in SRAM. */ |
| 1605 | EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_TX_DC_BASE_ADR, |
| 1606 | efx->type->tx_dc_base / 8); |
| 1607 | efx_writeo(efx, &temp, FR_AZ_SRM_TX_DC_CFG); |
| 1608 | EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_RX_DC_BASE_ADR, |
| 1609 | efx->type->rx_dc_base / 8); |
| 1610 | efx_writeo(efx, &temp, FR_AZ_SRM_RX_DC_CFG); |
| 1611 | |
| 1612 | /* Set TX descriptor cache size. */ |
| 1613 | BUILD_BUG_ON(TX_DC_ENTRIES != (8 << TX_DC_ENTRIES_ORDER)); |
| 1614 | EFX_POPULATE_OWORD_1(temp, FRF_AZ_TX_DC_SIZE, TX_DC_ENTRIES_ORDER); |
| 1615 | efx_writeo(efx, &temp, FR_AZ_TX_DC_CFG); |
| 1616 | |
| 1617 | /* Set RX descriptor cache size. Set low watermark to size-8, as |
| 1618 | * this allows most efficient prefetching. |
| 1619 | */ |
| 1620 | BUILD_BUG_ON(RX_DC_ENTRIES != (8 << RX_DC_ENTRIES_ORDER)); |
| 1621 | EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_SIZE, RX_DC_ENTRIES_ORDER); |
| 1622 | efx_writeo(efx, &temp, FR_AZ_RX_DC_CFG); |
| 1623 | EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_PF_LWM, RX_DC_ENTRIES - 8); |
| 1624 | efx_writeo(efx, &temp, FR_AZ_RX_DC_PF_WM); |
| 1625 | |
| 1626 | /* Program INT_KER address */ |
| 1627 | EFX_POPULATE_OWORD_2(temp, |
| 1628 | FRF_AZ_NORM_INT_VEC_DIS_KER, |
| 1629 | EFX_INT_MODE_USE_MSI(efx), |
| 1630 | FRF_AZ_INT_ADR_KER, efx->irq_status.dma_addr); |
| 1631 | efx_writeo(efx, &temp, FR_AZ_INT_ADR_KER); |
| 1632 | |
Steve Hodgson | 6369545 | 2010-04-28 09:27:36 +0000 | [diff] [blame] | 1633 | if (EFX_WORKAROUND_17213(efx) && !EFX_INT_MODE_USE_MSI(efx)) |
| 1634 | /* Use an interrupt level unused by event queues */ |
Ben Hutchings | 1646a6f | 2012-01-05 20:14:10 +0000 | [diff] [blame] | 1635 | efx->irq_level = 0x1f; |
Steve Hodgson | 6369545 | 2010-04-28 09:27:36 +0000 | [diff] [blame] | 1636 | else |
| 1637 | /* Use a valid MSI-X vector */ |
Ben Hutchings | 1646a6f | 2012-01-05 20:14:10 +0000 | [diff] [blame] | 1638 | efx->irq_level = 0; |
Steve Hodgson | 6369545 | 2010-04-28 09:27:36 +0000 | [diff] [blame] | 1639 | |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1640 | /* Enable all the genuinely fatal interrupts. (They are still |
| 1641 | * masked by the overall interrupt mask, controlled by |
| 1642 | * falcon_interrupts()). |
| 1643 | * |
| 1644 | * Note: All other fatal interrupts are enabled |
| 1645 | */ |
| 1646 | EFX_POPULATE_OWORD_3(temp, |
| 1647 | FRF_AZ_ILL_ADR_INT_KER_EN, 1, |
| 1648 | FRF_AZ_RBUF_OWN_INT_KER_EN, 1, |
| 1649 | FRF_AZ_TBUF_OWN_INT_KER_EN, 1); |
Steve Hodgson | b17424b | 2010-04-28 09:25:22 +0000 | [diff] [blame] | 1650 | if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) |
| 1651 | EFX_SET_OWORD_FIELD(temp, FRF_CZ_SRAM_PERR_INT_P_KER_EN, 1); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1652 | EFX_INVERT_OWORD(temp); |
| 1653 | efx_writeo(efx, &temp, FR_AZ_FATAL_INTR_KER); |
| 1654 | |
Ben Hutchings | 765c9f4 | 2010-06-30 05:06:28 +0000 | [diff] [blame] | 1655 | efx_nic_push_rx_indir_table(efx); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1656 | |
| 1657 | /* Disable the ugly timer-based TX DMA backoff and allow TX DMA to be |
| 1658 | * controlled by the RX FIFO fill level. Set arbitration to one pkt/Q. |
| 1659 | */ |
| 1660 | efx_reado(efx, &temp, FR_AZ_TX_RESERVED); |
| 1661 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER, 0xfe); |
| 1662 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER_EN, 1); |
| 1663 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_ONE_PKT_PER_Q, 1); |
Ben Hutchings | cd38557 | 2010-11-15 23:53:11 +0000 | [diff] [blame] | 1664 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PUSH_EN, 1); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1665 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_DIS_NON_IP_EV, 1); |
| 1666 | /* Enable SW_EV to inherit in char driver - assume harmless here */ |
| 1667 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_SOFT_EVT_EN, 1); |
| 1668 | /* Prefetch threshold 2 => fetch when descriptor cache half empty */ |
| 1669 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_THRESHOLD, 2); |
Ben Hutchings | 286d47b | 2009-12-23 13:49:13 +0000 | [diff] [blame] | 1670 | /* Disable hardware watchdog which can misfire */ |
| 1671 | EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1672 | /* Squash TX of packets of 16 bytes or less */ |
| 1673 | if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) |
| 1674 | EFX_SET_OWORD_FIELD(temp, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); |
| 1675 | efx_writeo(efx, &temp, FR_AZ_TX_RESERVED); |
Ben Hutchings | 94b274b | 2011-01-10 21:18:20 +0000 | [diff] [blame] | 1676 | |
| 1677 | if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) { |
| 1678 | EFX_POPULATE_OWORD_4(temp, |
| 1679 | /* Default values */ |
| 1680 | FRF_BZ_TX_PACE_SB_NOT_AF, 0x15, |
| 1681 | FRF_BZ_TX_PACE_SB_AF, 0xb, |
| 1682 | FRF_BZ_TX_PACE_FB_BASE, 0, |
| 1683 | /* Allow large pace values in the |
| 1684 | * fast bin. */ |
| 1685 | FRF_BZ_TX_PACE_BIN_TH, |
| 1686 | FFE_BZ_TX_PACE_RESERVED); |
| 1687 | efx_writeo(efx, &temp, FR_BZ_TX_PACE); |
| 1688 | } |
Ben Hutchings | 8e730c1 | 2009-11-29 15:14:45 +0000 | [diff] [blame] | 1689 | } |
Ben Hutchings | 5b98c1b | 2010-06-21 03:06:53 +0000 | [diff] [blame] | 1690 | |
| 1691 | /* Register dump */ |
| 1692 | |
| 1693 | #define REGISTER_REVISION_A 1 |
| 1694 | #define REGISTER_REVISION_B 2 |
| 1695 | #define REGISTER_REVISION_C 3 |
| 1696 | #define REGISTER_REVISION_Z 3 /* latest revision */ |
| 1697 | |
| 1698 | struct efx_nic_reg { |
| 1699 | u32 offset:24; |
| 1700 | u32 min_revision:2, max_revision:2; |
| 1701 | }; |
| 1702 | |
| 1703 | #define REGISTER(name, min_rev, max_rev) { \ |
| 1704 | FR_ ## min_rev ## max_rev ## _ ## name, \ |
| 1705 | REGISTER_REVISION_ ## min_rev, REGISTER_REVISION_ ## max_rev \ |
| 1706 | } |
| 1707 | #define REGISTER_AA(name) REGISTER(name, A, A) |
| 1708 | #define REGISTER_AB(name) REGISTER(name, A, B) |
| 1709 | #define REGISTER_AZ(name) REGISTER(name, A, Z) |
| 1710 | #define REGISTER_BB(name) REGISTER(name, B, B) |
| 1711 | #define REGISTER_BZ(name) REGISTER(name, B, Z) |
| 1712 | #define REGISTER_CZ(name) REGISTER(name, C, Z) |
| 1713 | |
| 1714 | static const struct efx_nic_reg efx_nic_regs[] = { |
| 1715 | REGISTER_AZ(ADR_REGION), |
| 1716 | REGISTER_AZ(INT_EN_KER), |
| 1717 | REGISTER_BZ(INT_EN_CHAR), |
| 1718 | REGISTER_AZ(INT_ADR_KER), |
| 1719 | REGISTER_BZ(INT_ADR_CHAR), |
| 1720 | /* INT_ACK_KER is WO */ |
| 1721 | /* INT_ISR0 is RC */ |
| 1722 | REGISTER_AZ(HW_INIT), |
| 1723 | REGISTER_CZ(USR_EV_CFG), |
| 1724 | REGISTER_AB(EE_SPI_HCMD), |
| 1725 | REGISTER_AB(EE_SPI_HADR), |
| 1726 | REGISTER_AB(EE_SPI_HDATA), |
| 1727 | REGISTER_AB(EE_BASE_PAGE), |
| 1728 | REGISTER_AB(EE_VPD_CFG0), |
| 1729 | /* EE_VPD_SW_CNTL and EE_VPD_SW_DATA are not used */ |
| 1730 | /* PMBX_DBG_IADDR and PBMX_DBG_IDATA are indirect */ |
| 1731 | /* PCIE_CORE_INDIRECT is indirect */ |
| 1732 | REGISTER_AB(NIC_STAT), |
| 1733 | REGISTER_AB(GPIO_CTL), |
| 1734 | REGISTER_AB(GLB_CTL), |
| 1735 | /* FATAL_INTR_KER and FATAL_INTR_CHAR are partly RC */ |
| 1736 | REGISTER_BZ(DP_CTRL), |
| 1737 | REGISTER_AZ(MEM_STAT), |
| 1738 | REGISTER_AZ(CS_DEBUG), |
| 1739 | REGISTER_AZ(ALTERA_BUILD), |
| 1740 | REGISTER_AZ(CSR_SPARE), |
| 1741 | REGISTER_AB(PCIE_SD_CTL0123), |
| 1742 | REGISTER_AB(PCIE_SD_CTL45), |
| 1743 | REGISTER_AB(PCIE_PCS_CTL_STAT), |
| 1744 | /* DEBUG_DATA_OUT is not used */ |
| 1745 | /* DRV_EV is WO */ |
| 1746 | REGISTER_AZ(EVQ_CTL), |
| 1747 | REGISTER_AZ(EVQ_CNT1), |
| 1748 | REGISTER_AZ(EVQ_CNT2), |
| 1749 | REGISTER_AZ(BUF_TBL_CFG), |
| 1750 | REGISTER_AZ(SRM_RX_DC_CFG), |
| 1751 | REGISTER_AZ(SRM_TX_DC_CFG), |
| 1752 | REGISTER_AZ(SRM_CFG), |
| 1753 | /* BUF_TBL_UPD is WO */ |
| 1754 | REGISTER_AZ(SRM_UPD_EVQ), |
| 1755 | REGISTER_AZ(SRAM_PARITY), |
| 1756 | REGISTER_AZ(RX_CFG), |
| 1757 | REGISTER_BZ(RX_FILTER_CTL), |
| 1758 | /* RX_FLUSH_DESCQ is WO */ |
| 1759 | REGISTER_AZ(RX_DC_CFG), |
| 1760 | REGISTER_AZ(RX_DC_PF_WM), |
| 1761 | REGISTER_BZ(RX_RSS_TKEY), |
| 1762 | /* RX_NODESC_DROP is RC */ |
| 1763 | REGISTER_AA(RX_SELF_RST), |
| 1764 | /* RX_DEBUG, RX_PUSH_DROP are not used */ |
| 1765 | REGISTER_CZ(RX_RSS_IPV6_REG1), |
| 1766 | REGISTER_CZ(RX_RSS_IPV6_REG2), |
| 1767 | REGISTER_CZ(RX_RSS_IPV6_REG3), |
| 1768 | /* TX_FLUSH_DESCQ is WO */ |
| 1769 | REGISTER_AZ(TX_DC_CFG), |
| 1770 | REGISTER_AA(TX_CHKSM_CFG), |
| 1771 | REGISTER_AZ(TX_CFG), |
| 1772 | /* TX_PUSH_DROP is not used */ |
| 1773 | REGISTER_AZ(TX_RESERVED), |
| 1774 | REGISTER_BZ(TX_PACE), |
| 1775 | /* TX_PACE_DROP_QID is RC */ |
| 1776 | REGISTER_BB(TX_VLAN), |
| 1777 | REGISTER_BZ(TX_IPFIL_PORTEN), |
| 1778 | REGISTER_AB(MD_TXD), |
| 1779 | REGISTER_AB(MD_RXD), |
| 1780 | REGISTER_AB(MD_CS), |
| 1781 | REGISTER_AB(MD_PHY_ADR), |
| 1782 | REGISTER_AB(MD_ID), |
| 1783 | /* MD_STAT is RC */ |
| 1784 | REGISTER_AB(MAC_STAT_DMA), |
| 1785 | REGISTER_AB(MAC_CTRL), |
| 1786 | REGISTER_BB(GEN_MODE), |
| 1787 | REGISTER_AB(MAC_MC_HASH_REG0), |
| 1788 | REGISTER_AB(MAC_MC_HASH_REG1), |
| 1789 | REGISTER_AB(GM_CFG1), |
| 1790 | REGISTER_AB(GM_CFG2), |
| 1791 | /* GM_IPG and GM_HD are not used */ |
| 1792 | REGISTER_AB(GM_MAX_FLEN), |
| 1793 | /* GM_TEST is not used */ |
| 1794 | REGISTER_AB(GM_ADR1), |
| 1795 | REGISTER_AB(GM_ADR2), |
| 1796 | REGISTER_AB(GMF_CFG0), |
| 1797 | REGISTER_AB(GMF_CFG1), |
| 1798 | REGISTER_AB(GMF_CFG2), |
| 1799 | REGISTER_AB(GMF_CFG3), |
| 1800 | REGISTER_AB(GMF_CFG4), |
| 1801 | REGISTER_AB(GMF_CFG5), |
| 1802 | REGISTER_BB(TX_SRC_MAC_CTL), |
| 1803 | REGISTER_AB(XM_ADR_LO), |
| 1804 | REGISTER_AB(XM_ADR_HI), |
| 1805 | REGISTER_AB(XM_GLB_CFG), |
| 1806 | REGISTER_AB(XM_TX_CFG), |
| 1807 | REGISTER_AB(XM_RX_CFG), |
| 1808 | REGISTER_AB(XM_MGT_INT_MASK), |
| 1809 | REGISTER_AB(XM_FC), |
| 1810 | REGISTER_AB(XM_PAUSE_TIME), |
| 1811 | REGISTER_AB(XM_TX_PARAM), |
| 1812 | REGISTER_AB(XM_RX_PARAM), |
| 1813 | /* XM_MGT_INT_MSK (note no 'A') is RC */ |
| 1814 | REGISTER_AB(XX_PWR_RST), |
| 1815 | REGISTER_AB(XX_SD_CTL), |
| 1816 | REGISTER_AB(XX_TXDRV_CTL), |
| 1817 | /* XX_PRBS_CTL, XX_PRBS_CHK and XX_PRBS_ERR are not used */ |
| 1818 | /* XX_CORE_STAT is partly RC */ |
| 1819 | }; |
| 1820 | |
| 1821 | struct efx_nic_reg_table { |
| 1822 | u32 offset:24; |
| 1823 | u32 min_revision:2, max_revision:2; |
| 1824 | u32 step:6, rows:21; |
| 1825 | }; |
| 1826 | |
| 1827 | #define REGISTER_TABLE_DIMENSIONS(_, offset, min_rev, max_rev, step, rows) { \ |
| 1828 | offset, \ |
| 1829 | REGISTER_REVISION_ ## min_rev, REGISTER_REVISION_ ## max_rev, \ |
| 1830 | step, rows \ |
| 1831 | } |
Ben Hutchings | 9c636ba | 2012-01-05 17:19:45 +0000 | [diff] [blame] | 1832 | #define REGISTER_TABLE(name, min_rev, max_rev) \ |
Ben Hutchings | 5b98c1b | 2010-06-21 03:06:53 +0000 | [diff] [blame] | 1833 | REGISTER_TABLE_DIMENSIONS( \ |
| 1834 | name, FR_ ## min_rev ## max_rev ## _ ## name, \ |
| 1835 | min_rev, max_rev, \ |
| 1836 | FR_ ## min_rev ## max_rev ## _ ## name ## _STEP, \ |
| 1837 | FR_ ## min_rev ## max_rev ## _ ## name ## _ROWS) |
| 1838 | #define REGISTER_TABLE_AA(name) REGISTER_TABLE(name, A, A) |
| 1839 | #define REGISTER_TABLE_AZ(name) REGISTER_TABLE(name, A, Z) |
| 1840 | #define REGISTER_TABLE_BB(name) REGISTER_TABLE(name, B, B) |
| 1841 | #define REGISTER_TABLE_BZ(name) REGISTER_TABLE(name, B, Z) |
| 1842 | #define REGISTER_TABLE_BB_CZ(name) \ |
| 1843 | REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, B, B, \ |
| 1844 | FR_BZ_ ## name ## _STEP, \ |
| 1845 | FR_BB_ ## name ## _ROWS), \ |
| 1846 | REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, C, Z, \ |
| 1847 | FR_BZ_ ## name ## _STEP, \ |
| 1848 | FR_CZ_ ## name ## _ROWS) |
| 1849 | #define REGISTER_TABLE_CZ(name) REGISTER_TABLE(name, C, Z) |
| 1850 | |
| 1851 | static const struct efx_nic_reg_table efx_nic_reg_tables[] = { |
| 1852 | /* DRIVER is not used */ |
| 1853 | /* EVQ_RPTR, TIMER_COMMAND, USR_EV and {RX,TX}_DESC_UPD are WO */ |
| 1854 | REGISTER_TABLE_BB(TX_IPFIL_TBL), |
| 1855 | REGISTER_TABLE_BB(TX_SRC_MAC_TBL), |
| 1856 | REGISTER_TABLE_AA(RX_DESC_PTR_TBL_KER), |
| 1857 | REGISTER_TABLE_BB_CZ(RX_DESC_PTR_TBL), |
| 1858 | REGISTER_TABLE_AA(TX_DESC_PTR_TBL_KER), |
| 1859 | REGISTER_TABLE_BB_CZ(TX_DESC_PTR_TBL), |
| 1860 | REGISTER_TABLE_AA(EVQ_PTR_TBL_KER), |
| 1861 | REGISTER_TABLE_BB_CZ(EVQ_PTR_TBL), |
Ben Hutchings | 75abc51 | 2010-09-20 08:43:53 +0000 | [diff] [blame] | 1862 | /* We can't reasonably read all of the buffer table (up to 8MB!). |
Ben Hutchings | 5b98c1b | 2010-06-21 03:06:53 +0000 | [diff] [blame] | 1863 | * However this driver will only use a few entries. Reading |
| 1864 | * 1K entries allows for some expansion of queue count and |
| 1865 | * size before we need to change the version. */ |
| 1866 | REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL_KER, FR_AA_BUF_FULL_TBL_KER, |
| 1867 | A, A, 8, 1024), |
| 1868 | REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL, FR_BZ_BUF_FULL_TBL, |
| 1869 | B, Z, 8, 1024), |
Ben Hutchings | 5b98c1b | 2010-06-21 03:06:53 +0000 | [diff] [blame] | 1870 | REGISTER_TABLE_CZ(RX_MAC_FILTER_TBL0), |
| 1871 | REGISTER_TABLE_BB_CZ(TIMER_TBL), |
| 1872 | REGISTER_TABLE_BB_CZ(TX_PACE_TBL), |
| 1873 | REGISTER_TABLE_BZ(RX_INDIRECTION_TBL), |
| 1874 | /* TX_FILTER_TBL0 is huge and not used by this driver */ |
| 1875 | REGISTER_TABLE_CZ(TX_MAC_FILTER_TBL0), |
| 1876 | REGISTER_TABLE_CZ(MC_TREG_SMEM), |
| 1877 | /* MSIX_PBA_TABLE is not mapped */ |
| 1878 | /* SRM_DBG is not mapped (and is redundant with BUF_FLL_TBL) */ |
Ben Hutchings | 75abc51 | 2010-09-20 08:43:53 +0000 | [diff] [blame] | 1879 | REGISTER_TABLE_BZ(RX_FILTER_TBL0), |
Ben Hutchings | 5b98c1b | 2010-06-21 03:06:53 +0000 | [diff] [blame] | 1880 | }; |
| 1881 | |
| 1882 | size_t efx_nic_get_regs_len(struct efx_nic *efx) |
| 1883 | { |
| 1884 | const struct efx_nic_reg *reg; |
| 1885 | const struct efx_nic_reg_table *table; |
| 1886 | size_t len = 0; |
| 1887 | |
| 1888 | for (reg = efx_nic_regs; |
| 1889 | reg < efx_nic_regs + ARRAY_SIZE(efx_nic_regs); |
| 1890 | reg++) |
| 1891 | if (efx->type->revision >= reg->min_revision && |
| 1892 | efx->type->revision <= reg->max_revision) |
| 1893 | len += sizeof(efx_oword_t); |
| 1894 | |
| 1895 | for (table = efx_nic_reg_tables; |
| 1896 | table < efx_nic_reg_tables + ARRAY_SIZE(efx_nic_reg_tables); |
| 1897 | table++) |
| 1898 | if (efx->type->revision >= table->min_revision && |
| 1899 | efx->type->revision <= table->max_revision) |
| 1900 | len += table->rows * min_t(size_t, table->step, 16); |
| 1901 | |
| 1902 | return len; |
| 1903 | } |
| 1904 | |
| 1905 | void efx_nic_get_regs(struct efx_nic *efx, void *buf) |
| 1906 | { |
| 1907 | const struct efx_nic_reg *reg; |
| 1908 | const struct efx_nic_reg_table *table; |
| 1909 | |
| 1910 | for (reg = efx_nic_regs; |
| 1911 | reg < efx_nic_regs + ARRAY_SIZE(efx_nic_regs); |
| 1912 | reg++) { |
| 1913 | if (efx->type->revision >= reg->min_revision && |
| 1914 | efx->type->revision <= reg->max_revision) { |
| 1915 | efx_reado(efx, (efx_oword_t *)buf, reg->offset); |
| 1916 | buf += sizeof(efx_oword_t); |
| 1917 | } |
| 1918 | } |
| 1919 | |
| 1920 | for (table = efx_nic_reg_tables; |
| 1921 | table < efx_nic_reg_tables + ARRAY_SIZE(efx_nic_reg_tables); |
| 1922 | table++) { |
| 1923 | size_t size, i; |
| 1924 | |
| 1925 | if (!(efx->type->revision >= table->min_revision && |
| 1926 | efx->type->revision <= table->max_revision)) |
| 1927 | continue; |
| 1928 | |
| 1929 | size = min_t(size_t, table->step, 16); |
| 1930 | |
| 1931 | for (i = 0; i < table->rows; i++) { |
| 1932 | switch (table->step) { |
| 1933 | case 4: /* 32-bit register or SRAM */ |
| 1934 | efx_readd_table(efx, buf, table->offset, i); |
| 1935 | break; |
| 1936 | case 8: /* 64-bit SRAM */ |
| 1937 | efx_sram_readq(efx, |
| 1938 | efx->membase + table->offset, |
| 1939 | buf, i); |
| 1940 | break; |
| 1941 | case 16: /* 128-bit register */ |
| 1942 | efx_reado_table(efx, buf, table->offset, i); |
| 1943 | break; |
| 1944 | case 32: /* 128-bit register, interleaved */ |
| 1945 | efx_reado_table(efx, buf, table->offset, 2 * i); |
| 1946 | break; |
| 1947 | default: |
| 1948 | WARN_ON(1); |
| 1949 | return; |
| 1950 | } |
| 1951 | buf += size; |
| 1952 | } |
| 1953 | } |
| 1954 | } |