Raghu Vatsavayi | f21fb3e | 2015-06-09 18:15:23 -0700 | [diff] [blame] | 1 | /********************************************************************** |
| 2 | * Author: Cavium, Inc. |
| 3 | * |
| 4 | * Contact: support@cavium.com |
| 5 | * Please include "LiquidIO" in the subject. |
| 6 | * |
| 7 | * Copyright (c) 2003-2015 Cavium, Inc. |
| 8 | * |
| 9 | * This file is free software; you can redistribute it and/or modify |
| 10 | * it under the terms of the GNU General Public License, Version 2, as |
| 11 | * published by the Free Software Foundation. |
| 12 | * |
| 13 | * This file is distributed in the hope that it will be useful, but |
| 14 | * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty |
| 15 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or |
| 16 | * NONINFRINGEMENT. See the GNU General Public License for more |
| 17 | * details. |
| 18 | * |
| 19 | * This file may also be available under a different license from Cavium. |
| 20 | * Contact Cavium, Inc. for more information |
| 21 | **********************************************************************/ |
| 22 | |
| 23 | /*! \file octeon_droq.h |
| 24 | * \brief Implementation of Octeon Output queues. "Output" is with |
| 25 | * respect to the Octeon device on the NIC. From this driver's point of |
| 26 | * view they are ingress queues. |
| 27 | */ |
| 28 | |
| 29 | #ifndef __OCTEON_DROQ_H__ |
| 30 | #define __OCTEON_DROQ_H__ |
| 31 | |
| 32 | /* Default number of packets that will be processed in one iteration. */ |
| 33 | #define MAX_PACKET_BUDGET 0xFFFFFFFF |
| 34 | |
| 35 | /** Octeon descriptor format. |
| 36 | * The descriptor ring is made of descriptors which have 2 64-bit values: |
| 37 | * -# Physical (bus) address of the data buffer. |
| 38 | * -# Physical (bus) address of a octeon_droq_info structure. |
| 39 | * The Octeon device DMA's incoming packets and its information at the address |
| 40 | * given by these descriptor fields. |
| 41 | */ |
| 42 | struct octeon_droq_desc { |
| 43 | /** The buffer pointer */ |
| 44 | u64 buffer_ptr; |
| 45 | |
| 46 | /** The Info pointer */ |
| 47 | u64 info_ptr; |
| 48 | }; |
| 49 | |
| 50 | #define OCT_DROQ_DESC_SIZE (sizeof(struct octeon_droq_desc)) |
| 51 | |
| 52 | /** Information about packet DMA'ed by Octeon. |
| 53 | * The format of the information available at Info Pointer after Octeon |
| 54 | * has posted a packet. Not all descriptors have valid information. Only |
| 55 | * the Info field of the first descriptor for a packet has information |
| 56 | * about the packet. |
| 57 | */ |
| 58 | struct octeon_droq_info { |
| 59 | /** The Output Receive Header. */ |
| 60 | union octeon_rh rh; |
| 61 | |
| 62 | /** The Length of the packet. */ |
| 63 | u64 length; |
| 64 | }; |
| 65 | |
| 66 | #define OCT_DROQ_INFO_SIZE (sizeof(struct octeon_droq_info)) |
| 67 | |
Raghu Vatsavayi | cabeb13 | 2016-06-14 16:54:47 -0700 | [diff] [blame] | 68 | struct octeon_skb_page_info { |
| 69 | /* DMA address for the page */ |
| 70 | dma_addr_t dma; |
| 71 | |
| 72 | /* Page for the rx dma **/ |
| 73 | struct page *page; |
| 74 | |
| 75 | /** which offset into page */ |
| 76 | unsigned int page_offset; |
| 77 | }; |
| 78 | |
Raghu Vatsavayi | f21fb3e | 2015-06-09 18:15:23 -0700 | [diff] [blame] | 79 | /** Pointer to data buffer. |
| 80 | * Driver keeps a pointer to the data buffer that it made available to |
| 81 | * the Octeon device. Since the descriptor ring keeps physical (bus) |
| 82 | * addresses, this field is required for the driver to keep track of |
| 83 | * the virtual address pointers. |
| 84 | */ |
| 85 | struct octeon_recv_buffer { |
| 86 | /** Packet buffer, including metadata. */ |
| 87 | void *buffer; |
| 88 | |
| 89 | /** Data in the packet buffer. */ |
| 90 | u8 *data; |
Raghu Vatsavayi | cabeb13 | 2016-06-14 16:54:47 -0700 | [diff] [blame] | 91 | |
| 92 | /** pg_info **/ |
| 93 | struct octeon_skb_page_info pg_info; |
Raghu Vatsavayi | f21fb3e | 2015-06-09 18:15:23 -0700 | [diff] [blame] | 94 | }; |
| 95 | |
| 96 | #define OCT_DROQ_RECVBUF_SIZE (sizeof(struct octeon_recv_buffer)) |
| 97 | |
| 98 | /** Output Queue statistics. Each output queue has four stats fields. */ |
| 99 | struct oct_droq_stats { |
| 100 | /** Number of packets received in this queue. */ |
| 101 | u64 pkts_received; |
| 102 | |
| 103 | /** Bytes received by this queue. */ |
| 104 | u64 bytes_received; |
| 105 | |
| 106 | /** Packets dropped due to no dispatch function. */ |
| 107 | u64 dropped_nodispatch; |
| 108 | |
| 109 | /** Packets dropped due to no memory available. */ |
| 110 | u64 dropped_nomem; |
| 111 | |
| 112 | /** Packets dropped due to large number of pkts to process. */ |
| 113 | u64 dropped_toomany; |
| 114 | |
| 115 | /** Number of packets sent to stack from this queue. */ |
| 116 | u64 rx_pkts_received; |
| 117 | |
| 118 | /** Number of Bytes sent to stack from this queue. */ |
| 119 | u64 rx_bytes_received; |
| 120 | |
| 121 | /** Num of Packets dropped due to receive path failures. */ |
| 122 | u64 rx_dropped; |
Raghu Vatsavayi | cabeb13 | 2016-06-14 16:54:47 -0700 | [diff] [blame] | 123 | |
Raghu Vatsavayi | 01fb237 | 2016-07-03 13:56:47 -0700 | [diff] [blame] | 124 | /** Num of vxlan packets received; */ |
| 125 | u64 rx_vxlan; |
| 126 | |
Raghu Vatsavayi | cabeb13 | 2016-06-14 16:54:47 -0700 | [diff] [blame] | 127 | /** Num of failures of recv_buffer_alloc() */ |
| 128 | u64 rx_alloc_failure; |
| 129 | |
Raghu Vatsavayi | f21fb3e | 2015-06-09 18:15:23 -0700 | [diff] [blame] | 130 | }; |
| 131 | |
| 132 | #define POLL_EVENT_INTR_ARRIVED 1 |
| 133 | #define POLL_EVENT_PROCESS_PKTS 2 |
| 134 | #define POLL_EVENT_PENDING_PKTS 3 |
| 135 | #define POLL_EVENT_ENABLE_INTR 4 |
| 136 | |
| 137 | /* The maximum number of buffers that can be dispatched from the |
| 138 | * output/dma queue. Set to 64 assuming 1K buffers in DROQ and the fact that |
| 139 | * max packet size from DROQ is 64K. |
| 140 | */ |
| 141 | #define MAX_RECV_BUFS 64 |
| 142 | |
| 143 | /** Receive Packet format used when dispatching output queue packets |
| 144 | * with non-raw opcodes. |
| 145 | * The received packet will be sent to the upper layers using this |
| 146 | * structure which is passed as a parameter to the dispatch function |
| 147 | */ |
| 148 | struct octeon_recv_pkt { |
| 149 | /** Number of buffers in this received packet */ |
| 150 | u16 buffer_count; |
| 151 | |
| 152 | /** Id of the device that is sending the packet up */ |
| 153 | u16 octeon_id; |
| 154 | |
| 155 | /** Length of data in the packet buffer */ |
| 156 | u32 length; |
| 157 | |
| 158 | /** The receive header */ |
| 159 | union octeon_rh rh; |
| 160 | |
| 161 | /** Pointer to the OS-specific packet buffer */ |
| 162 | void *buffer_ptr[MAX_RECV_BUFS]; |
| 163 | |
| 164 | /** Size of the buffers pointed to by ptr's in buffer_ptr */ |
| 165 | u32 buffer_size[MAX_RECV_BUFS]; |
| 166 | }; |
| 167 | |
| 168 | #define OCT_RECV_PKT_SIZE (sizeof(struct octeon_recv_pkt)) |
| 169 | |
| 170 | /** The first parameter of a dispatch function. |
| 171 | * For a raw mode opcode, the driver dispatches with the device |
| 172 | * pointer in this structure. |
| 173 | * For non-raw mode opcode, the driver dispatches the recv_pkt |
| 174 | * created to contain the buffers with data received from Octeon. |
| 175 | * --------------------- |
| 176 | * | *recv_pkt ----|--- |
| 177 | * |-------------------| | |
| 178 | * | 0 or more bytes | | |
| 179 | * | reserved by driver| | |
| 180 | * |-------------------|<-/ |
| 181 | * | octeon_recv_pkt | |
| 182 | * | | |
| 183 | * |___________________| |
| 184 | */ |
| 185 | struct octeon_recv_info { |
| 186 | void *rsvd; |
| 187 | struct octeon_recv_pkt *recv_pkt; |
| 188 | }; |
| 189 | |
| 190 | #define OCT_RECV_INFO_SIZE (sizeof(struct octeon_recv_info)) |
| 191 | |
| 192 | /** Allocate a recv_info structure. The recv_pkt pointer in the recv_info |
| 193 | * structure is filled in before this call returns. |
| 194 | * @param extra_bytes - extra bytes to be allocated at the end of the recv info |
| 195 | * structure. |
| 196 | * @return - pointer to a newly allocated recv_info structure. |
| 197 | */ |
| 198 | static inline struct octeon_recv_info *octeon_alloc_recv_info(int extra_bytes) |
| 199 | { |
| 200 | struct octeon_recv_info *recv_info; |
| 201 | u8 *buf; |
| 202 | |
| 203 | buf = kmalloc(OCT_RECV_PKT_SIZE + OCT_RECV_INFO_SIZE + |
| 204 | extra_bytes, GFP_ATOMIC); |
| 205 | if (!buf) |
| 206 | return NULL; |
| 207 | |
| 208 | recv_info = (struct octeon_recv_info *)buf; |
| 209 | recv_info->recv_pkt = |
| 210 | (struct octeon_recv_pkt *)(buf + OCT_RECV_INFO_SIZE); |
| 211 | recv_info->rsvd = NULL; |
| 212 | if (extra_bytes) |
| 213 | recv_info->rsvd = buf + OCT_RECV_INFO_SIZE + OCT_RECV_PKT_SIZE; |
| 214 | |
| 215 | return recv_info; |
| 216 | } |
| 217 | |
| 218 | /** Free a recv_info structure. |
| 219 | * @param recv_info - Pointer to receive_info to be freed |
| 220 | */ |
| 221 | static inline void octeon_free_recv_info(struct octeon_recv_info *recv_info) |
| 222 | { |
| 223 | kfree(recv_info); |
| 224 | } |
| 225 | |
| 226 | typedef int (*octeon_dispatch_fn_t)(struct octeon_recv_info *, void *); |
| 227 | |
| 228 | /** Used by NIC module to register packet handler and to get device |
| 229 | * information for each octeon device. |
| 230 | */ |
| 231 | struct octeon_droq_ops { |
| 232 | /** This registered function will be called by the driver with |
| 233 | * the octeon id, pointer to buffer from droq and length of |
| 234 | * data in the buffer. The receive header gives the port |
| 235 | * number to the caller. Function pointer is set by caller. |
| 236 | */ |
Raghu Vatsavayi | 0cece6c | 2016-06-14 16:54:50 -0700 | [diff] [blame] | 237 | void (*fptr)(u32, void *, u32, union octeon_rh *, void *, void *); |
| 238 | void *farg; |
Raghu Vatsavayi | f21fb3e | 2015-06-09 18:15:23 -0700 | [diff] [blame] | 239 | |
| 240 | /* This function will be called by the driver for all NAPI related |
| 241 | * events. The first param is the octeon id. The second param is the |
| 242 | * output queue number. The third is the NAPI event that occurred. |
| 243 | */ |
| 244 | void (*napi_fn)(void *); |
| 245 | |
| 246 | u32 poll_mode; |
| 247 | |
| 248 | /** Flag indicating if the DROQ handler should drop packets that |
| 249 | * it cannot handle in one iteration. Set by caller. |
| 250 | */ |
| 251 | u32 drop_on_max; |
| 252 | }; |
| 253 | |
| 254 | /** The Descriptor Ring Output Queue structure. |
| 255 | * This structure has all the information required to implement a |
| 256 | * Octeon DROQ. |
| 257 | */ |
| 258 | struct octeon_droq { |
| 259 | /** A spinlock to protect access to this ring. */ |
| 260 | spinlock_t lock; |
| 261 | |
| 262 | u32 q_no; |
| 263 | |
Raghu Vatsavayi | cd8b1eb | 2016-08-31 11:03:22 -0700 | [diff] [blame] | 264 | u32 pkt_count; |
| 265 | |
Raghu Vatsavayi | f21fb3e | 2015-06-09 18:15:23 -0700 | [diff] [blame] | 266 | struct octeon_droq_ops ops; |
| 267 | |
| 268 | struct octeon_device *oct_dev; |
| 269 | |
| 270 | /** The 8B aligned descriptor ring starts at this address. */ |
| 271 | struct octeon_droq_desc *desc_ring; |
| 272 | |
| 273 | /** Index in the ring where the driver should read the next packet */ |
| 274 | u32 read_idx; |
| 275 | |
| 276 | /** Index in the ring where Octeon will write the next packet */ |
| 277 | u32 write_idx; |
| 278 | |
| 279 | /** Index in the ring where the driver will refill the descriptor's |
| 280 | * buffer |
| 281 | */ |
| 282 | u32 refill_idx; |
| 283 | |
| 284 | /** Packets pending to be processed */ |
| 285 | atomic_t pkts_pending; |
| 286 | |
| 287 | /** Number of descriptors in this ring. */ |
| 288 | u32 max_count; |
| 289 | |
| 290 | /** The number of descriptors pending refill. */ |
| 291 | u32 refill_count; |
| 292 | |
| 293 | u32 pkts_per_intr; |
| 294 | u32 refill_threshold; |
| 295 | |
| 296 | /** The max number of descriptors in DROQ without a buffer. |
| 297 | * This field is used to keep track of empty space threshold. If the |
| 298 | * refill_count reaches this value, the DROQ cannot accept a max-sized |
| 299 | * (64K) packet. |
| 300 | */ |
| 301 | u32 max_empty_descs; |
| 302 | |
| 303 | /** The 8B aligned info ptrs begin from this address. */ |
| 304 | struct octeon_droq_info *info_list; |
| 305 | |
| 306 | /** The receive buffer list. This list has the virtual addresses of the |
| 307 | * buffers. |
| 308 | */ |
| 309 | struct octeon_recv_buffer *recv_buf_list; |
| 310 | |
| 311 | /** The size of each buffer pointed by the buffer pointer. */ |
| 312 | u32 buffer_size; |
| 313 | |
| 314 | /** Pointer to the mapped packet credit register. |
| 315 | * Host writes number of info/buffer ptrs available to this register |
| 316 | */ |
| 317 | void __iomem *pkts_credit_reg; |
| 318 | |
| 319 | /** Pointer to the mapped packet sent register. |
| 320 | * Octeon writes the number of packets DMA'ed to host memory |
| 321 | * in this register. |
| 322 | */ |
| 323 | void __iomem *pkts_sent_reg; |
| 324 | |
| 325 | struct list_head dispatch_list; |
| 326 | |
| 327 | /** Statistics for this DROQ. */ |
| 328 | struct oct_droq_stats stats; |
| 329 | |
| 330 | /** DMA mapped address of the DROQ descriptor ring. */ |
| 331 | size_t desc_ring_dma; |
| 332 | |
| 333 | /** Info ptr list are allocated at this virtual address. */ |
| 334 | size_t info_base_addr; |
| 335 | |
| 336 | /** DMA mapped address of the info list */ |
| 337 | size_t info_list_dma; |
| 338 | |
| 339 | /** Allocated size of info list. */ |
| 340 | u32 info_alloc_size; |
| 341 | |
| 342 | /** application context */ |
| 343 | void *app_ctx; |
| 344 | |
| 345 | struct napi_struct napi; |
| 346 | |
| 347 | u32 cpu_id; |
| 348 | |
| 349 | struct call_single_data csd; |
| 350 | }; |
| 351 | |
| 352 | #define OCT_DROQ_SIZE (sizeof(struct octeon_droq)) |
| 353 | |
| 354 | /** |
| 355 | * Allocates space for the descriptor ring for the droq and sets the |
| 356 | * base addr, num desc etc in Octeon registers. |
| 357 | * |
| 358 | * @param oct_dev - pointer to the octeon device structure |
| 359 | * @param q_no - droq no. ranges from 0 - 3. |
| 360 | * @param app_ctx - pointer to application context |
| 361 | * @return Success: 0 Failure: 1 |
| 362 | */ |
| 363 | int octeon_init_droq(struct octeon_device *oct_dev, |
| 364 | u32 q_no, |
| 365 | u32 num_descs, |
| 366 | u32 desc_size, |
| 367 | void *app_ctx); |
| 368 | |
| 369 | /** |
| 370 | * Frees the space for descriptor ring for the droq. |
| 371 | * |
| 372 | * @param oct_dev - pointer to the octeon device structure |
| 373 | * @param q_no - droq no. ranges from 0 - 3. |
| 374 | * @return: Success: 0 Failure: 1 |
| 375 | */ |
| 376 | int octeon_delete_droq(struct octeon_device *oct_dev, u32 q_no); |
| 377 | |
| 378 | /** Register a change in droq operations. The ops field has a pointer to a |
| 379 | * function which will called by the DROQ handler for all packets arriving |
| 380 | * on output queues given by q_no irrespective of the type of packet. |
| 381 | * The ops field also has a flag which if set tells the DROQ handler to |
| 382 | * drop packets if it receives more than what it can process in one |
| 383 | * invocation of the handler. |
| 384 | * @param oct - octeon device |
| 385 | * @param q_no - octeon output queue number (0 <= q_no <= MAX_OCTEON_DROQ-1 |
| 386 | * @param ops - the droq_ops settings for this queue |
| 387 | * @return - 0 on success, -ENODEV or -EINVAL on error. |
| 388 | */ |
| 389 | int |
| 390 | octeon_register_droq_ops(struct octeon_device *oct, |
| 391 | u32 q_no, |
| 392 | struct octeon_droq_ops *ops); |
| 393 | |
| 394 | /** Resets the function pointer and flag settings made by |
| 395 | * octeon_register_droq_ops(). After this routine is called, the DROQ handler |
| 396 | * will lookup dispatch function for each arriving packet on the output queue |
| 397 | * given by q_no. |
| 398 | * @param oct - octeon device |
| 399 | * @param q_no - octeon output queue number (0 <= q_no <= MAX_OCTEON_DROQ-1 |
| 400 | * @return - 0 on success, -ENODEV or -EINVAL on error. |
| 401 | */ |
| 402 | int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no); |
| 403 | |
| 404 | /** Register a dispatch function for a opcode/subcode. The driver will call |
| 405 | * this dispatch function when it receives a packet with the given |
| 406 | * opcode/subcode in its output queues along with the user specified |
| 407 | * argument. |
| 408 | * @param oct - the octeon device to register with. |
| 409 | * @param opcode - the opcode for which the dispatch will be registered. |
| 410 | * @param subcode - the subcode for which the dispatch will be registered |
| 411 | * @param fn - the dispatch function. |
| 412 | * @param fn_arg - user specified that will be passed along with the |
| 413 | * dispatch function by the driver. |
| 414 | * @return Success: 0; Failure: 1 |
| 415 | */ |
| 416 | int octeon_register_dispatch_fn(struct octeon_device *oct, |
| 417 | u16 opcode, |
| 418 | u16 subcode, |
| 419 | octeon_dispatch_fn_t fn, void *fn_arg); |
| 420 | |
Raghu Vatsavayi | f21fb3e | 2015-06-09 18:15:23 -0700 | [diff] [blame] | 421 | void octeon_droq_print_stats(void); |
| 422 | |
Raghu Vatsavayi | a7d5a3d | 2016-07-03 13:56:48 -0700 | [diff] [blame] | 423 | u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq); |
Raghu Vatsavayi | f21fb3e | 2015-06-09 18:15:23 -0700 | [diff] [blame] | 424 | |
| 425 | int octeon_create_droq(struct octeon_device *oct, u32 q_no, |
| 426 | u32 num_descs, u32 desc_size, void *app_ctx); |
| 427 | |
| 428 | int octeon_droq_process_packets(struct octeon_device *oct, |
| 429 | struct octeon_droq *droq, |
| 430 | u32 budget); |
| 431 | |
| 432 | int octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, |
| 433 | int cmd, u32 arg); |
| 434 | |
| 435 | #endif /*__OCTEON_DROQ_H__ */ |