Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 1 | #ifndef _PIO_H |
| 2 | #define _PIO_H |
| 3 | /* |
| 4 | * |
| 5 | * This file is provided under a dual BSD/GPLv2 license. When using or |
| 6 | * redistributing this file, you may do so under either license. |
| 7 | * |
| 8 | * GPL LICENSE SUMMARY |
| 9 | * |
| 10 | * Copyright(c) 2015 Intel Corporation. |
| 11 | * |
| 12 | * This program is free software; you can redistribute it and/or modify |
| 13 | * it under the terms of version 2 of the GNU General Public License as |
| 14 | * published by the Free Software Foundation. |
| 15 | * |
| 16 | * This program is distributed in the hope that it will be useful, but |
| 17 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
| 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 19 | * General Public License for more details. |
| 20 | * |
| 21 | * BSD LICENSE |
| 22 | * |
| 23 | * Copyright(c) 2015 Intel Corporation. |
| 24 | * |
| 25 | * Redistribution and use in source and binary forms, with or without |
| 26 | * modification, are permitted provided that the following conditions |
| 27 | * are met: |
| 28 | * |
| 29 | * - Redistributions of source code must retain the above copyright |
| 30 | * notice, this list of conditions and the following disclaimer. |
| 31 | * - Redistributions in binary form must reproduce the above copyright |
| 32 | * notice, this list of conditions and the following disclaimer in |
| 33 | * the documentation and/or other materials provided with the |
| 34 | * distribution. |
| 35 | * - Neither the name of Intel Corporation nor the names of its |
| 36 | * contributors may be used to endorse or promote products derived |
| 37 | * from this software without specific prior written permission. |
| 38 | * |
| 39 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 40 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 41 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 42 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 43 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 44 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 45 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 46 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 47 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 48 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 49 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 50 | * |
| 51 | */ |
| 52 | |
| 53 | |
| 54 | /* send context types */ |
| 55 | #define SC_KERNEL 0 |
| 56 | #define SC_ACK 1 |
| 57 | #define SC_USER 2 |
| 58 | #define SC_MAX 3 |
| 59 | |
| 60 | /* invalid send context index */ |
| 61 | #define INVALID_SCI 0xff |
| 62 | |
| 63 | /* PIO buffer release callback function */ |
| 64 | typedef void (*pio_release_cb)(void *arg, int code); |
| 65 | |
| 66 | /* PIO release codes - in bits, as there could more than one that apply */ |
| 67 | #define PRC_OK 0 /* no known error */ |
| 68 | #define PRC_STATUS_ERR 0x01 /* credit return due to status error */ |
| 69 | #define PRC_PBC 0x02 /* credit return due to PBC */ |
| 70 | #define PRC_THRESHOLD 0x04 /* credit return due to threshold */ |
| 71 | #define PRC_FILL_ERR 0x08 /* credit return due fill error */ |
| 72 | #define PRC_FORCE 0x10 /* credit return due credit force */ |
| 73 | #define PRC_SC_DISABLE 0x20 /* clean-up after a context disable */ |
| 74 | |
| 75 | /* byte helper */ |
| 76 | union mix { |
| 77 | u64 val64; |
| 78 | u32 val32[2]; |
| 79 | u8 val8[8]; |
| 80 | }; |
| 81 | |
| 82 | /* an allocated PIO buffer */ |
| 83 | struct pio_buf { |
| 84 | struct send_context *sc;/* back pointer to owning send context */ |
| 85 | pio_release_cb cb; /* called when the buffer is released */ |
| 86 | void *arg; /* argument for cb */ |
| 87 | void __iomem *start; /* buffer start address */ |
| 88 | void __iomem *end; /* context end address */ |
| 89 | unsigned long size; /* context size, in bytes */ |
| 90 | unsigned long sent_at; /* buffer is sent when <= free */ |
| 91 | u32 block_count; /* size of buffer, in blocks */ |
| 92 | u32 qw_written; /* QW written so far */ |
| 93 | u32 carry_bytes; /* number of valid bytes in carry */ |
| 94 | union mix carry; /* pending unwritten bytes */ |
| 95 | }; |
| 96 | |
| 97 | /* cache line aligned pio buffer array */ |
| 98 | union pio_shadow_ring { |
| 99 | struct pio_buf pbuf; |
| 100 | u64 unused[16]; /* cache line spacer */ |
| 101 | } ____cacheline_aligned; |
| 102 | |
| 103 | /* per-NUMA send context */ |
| 104 | struct send_context { |
| 105 | /* read-only after init */ |
| 106 | struct hfi1_devdata *dd; /* device */ |
| 107 | void __iomem *base_addr; /* start of PIO memory */ |
| 108 | union pio_shadow_ring *sr; /* shadow ring */ |
| 109 | volatile __le64 *hw_free; /* HW free counter */ |
| 110 | struct work_struct halt_work; /* halted context work queue entry */ |
| 111 | unsigned long flags; /* flags */ |
| 112 | int node; /* context home node */ |
| 113 | int type; /* context type */ |
| 114 | u32 sw_index; /* software index number */ |
| 115 | u32 hw_context; /* hardware context number */ |
| 116 | u32 credits; /* number of blocks in context */ |
| 117 | u32 sr_size; /* size of the shadow ring */ |
| 118 | u32 group; /* credit return group */ |
| 119 | /* allocator fields */ |
| 120 | spinlock_t alloc_lock ____cacheline_aligned_in_smp; |
| 121 | unsigned long fill; /* official alloc count */ |
| 122 | unsigned long alloc_free; /* copy of free (less cache thrash) */ |
| 123 | u32 sr_head; /* shadow ring head */ |
| 124 | /* releaser fields */ |
| 125 | spinlock_t release_lock ____cacheline_aligned_in_smp; |
| 126 | unsigned long free; /* official free count */ |
| 127 | u32 sr_tail; /* shadow ring tail */ |
| 128 | /* list for PIO waiters */ |
| 129 | struct list_head piowait ____cacheline_aligned_in_smp; |
| 130 | spinlock_t credit_ctrl_lock ____cacheline_aligned_in_smp; |
| 131 | u64 credit_ctrl; /* cache for credit control */ |
| 132 | u32 credit_intr_count; /* count of credit intr users */ |
Mike Marciniszyn | a054374 | 2015-12-07 15:39:22 -0500 | [diff] [blame] | 133 | u32 __percpu *buffers_allocated;/* count of buffers allocated */ |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 134 | wait_queue_head_t halt_wait; /* wait until kernel sees interrupt */ |
| 135 | }; |
| 136 | |
| 137 | /* send context flags */ |
| 138 | #define SCF_ENABLED 0x01 |
| 139 | #define SCF_IN_FREE 0x02 |
| 140 | #define SCF_HALTED 0x04 |
| 141 | #define SCF_FROZEN 0x08 |
| 142 | |
| 143 | struct send_context_info { |
| 144 | struct send_context *sc; /* allocated working context */ |
| 145 | u16 allocated; /* has this been allocated? */ |
| 146 | u16 type; /* context type */ |
| 147 | u16 base; /* base in PIO array */ |
| 148 | u16 credits; /* size in PIO array */ |
| 149 | }; |
| 150 | |
| 151 | /* DMA credit return, index is always (context & 0x7) */ |
| 152 | struct credit_return { |
| 153 | volatile __le64 cr[8]; |
| 154 | }; |
| 155 | |
| 156 | /* NUMA indexed credit return array */ |
| 157 | struct credit_return_base { |
| 158 | struct credit_return *va; |
| 159 | dma_addr_t pa; |
| 160 | }; |
| 161 | |
| 162 | /* send context configuration sizes (one per type) */ |
| 163 | struct sc_config_sizes { |
| 164 | short int size; |
| 165 | short int count; |
| 166 | }; |
| 167 | |
| 168 | /* send context functions */ |
| 169 | int init_credit_return(struct hfi1_devdata *dd); |
| 170 | void free_credit_return(struct hfi1_devdata *dd); |
| 171 | int init_sc_pools_and_sizes(struct hfi1_devdata *dd); |
| 172 | int init_send_contexts(struct hfi1_devdata *dd); |
| 173 | int init_credit_return(struct hfi1_devdata *dd); |
| 174 | int init_pervl_scs(struct hfi1_devdata *dd); |
| 175 | struct send_context *sc_alloc(struct hfi1_devdata *dd, int type, |
| 176 | uint hdrqentsize, int numa); |
| 177 | void sc_free(struct send_context *sc); |
| 178 | int sc_enable(struct send_context *sc); |
| 179 | void sc_disable(struct send_context *sc); |
| 180 | int sc_restart(struct send_context *sc); |
| 181 | void sc_return_credits(struct send_context *sc); |
| 182 | void sc_flush(struct send_context *sc); |
| 183 | void sc_drop(struct send_context *sc); |
| 184 | void sc_stop(struct send_context *sc, int bit); |
| 185 | struct pio_buf *sc_buffer_alloc(struct send_context *sc, u32 dw_len, |
| 186 | pio_release_cb cb, void *arg); |
| 187 | void sc_release_update(struct send_context *sc); |
| 188 | void sc_return_credits(struct send_context *sc); |
| 189 | void sc_group_release_update(struct hfi1_devdata *dd, u32 hw_context); |
| 190 | void sc_add_credit_return_intr(struct send_context *sc); |
| 191 | void sc_del_credit_return_intr(struct send_context *sc); |
| 192 | void sc_set_cr_threshold(struct send_context *sc, u32 new_threshold); |
| 193 | u32 sc_mtu_to_threshold(struct send_context *sc, u32 mtu, u32 hdrqentsize); |
| 194 | void hfi1_sc_wantpiobuf_intr(struct send_context *sc, u32 needint); |
| 195 | void sc_wait(struct hfi1_devdata *dd); |
| 196 | void set_pio_integrity(struct send_context *sc); |
| 197 | |
| 198 | /* support functions */ |
| 199 | void pio_reset_all(struct hfi1_devdata *dd); |
| 200 | void pio_freeze(struct hfi1_devdata *dd); |
| 201 | void pio_kernel_unfreeze(struct hfi1_devdata *dd); |
| 202 | |
| 203 | /* global PIO send control operations */ |
| 204 | #define PSC_GLOBAL_ENABLE 0 |
| 205 | #define PSC_GLOBAL_DISABLE 1 |
| 206 | #define PSC_GLOBAL_VLARB_ENABLE 2 |
| 207 | #define PSC_GLOBAL_VLARB_DISABLE 3 |
| 208 | #define PSC_CM_RESET 4 |
| 209 | #define PSC_DATA_VL_ENABLE 5 |
| 210 | #define PSC_DATA_VL_DISABLE 6 |
| 211 | |
| 212 | void __cm_reset(struct hfi1_devdata *dd, u64 sendctrl); |
| 213 | void pio_send_control(struct hfi1_devdata *dd, int op); |
| 214 | |
| 215 | |
| 216 | /* PIO copy routines */ |
| 217 | void pio_copy(struct hfi1_devdata *dd, struct pio_buf *pbuf, u64 pbc, |
| 218 | const void *from, size_t count); |
| 219 | void seg_pio_copy_start(struct pio_buf *pbuf, u64 pbc, |
| 220 | const void *from, size_t nbytes); |
| 221 | void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes); |
| 222 | void seg_pio_copy_end(struct pio_buf *pbuf); |
| 223 | |
| 224 | #endif /* _PIO_H */ |