Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 1 | /* |
| 2 | * |
| 3 | * This file is provided under a dual BSD/GPLv2 license. When using or |
| 4 | * redistributing this file, you may do so under either license. |
| 5 | * |
| 6 | * GPL LICENSE SUMMARY |
| 7 | * |
| 8 | * Copyright(c) 2015 Intel Corporation. |
| 9 | * |
| 10 | * This program is free software; you can redistribute it and/or modify |
| 11 | * it under the terms of version 2 of the GNU General Public License as |
| 12 | * published by the Free Software Foundation. |
| 13 | * |
| 14 | * This program is distributed in the hope that it will be useful, but |
| 15 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
| 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 17 | * General Public License for more details. |
| 18 | * |
| 19 | * BSD LICENSE |
| 20 | * |
| 21 | * Copyright(c) 2015 Intel Corporation. |
| 22 | * |
| 23 | * Redistribution and use in source and binary forms, with or without |
| 24 | * modification, are permitted provided that the following conditions |
| 25 | * are met: |
| 26 | * |
| 27 | * - Redistributions of source code must retain the above copyright |
| 28 | * notice, this list of conditions and the following disclaimer. |
| 29 | * - Redistributions in binary form must reproduce the above copyright |
| 30 | * notice, this list of conditions and the following disclaimer in |
| 31 | * the documentation and/or other materials provided with the |
| 32 | * distribution. |
| 33 | * - Neither the name of Intel Corporation nor the names of its |
| 34 | * contributors may be used to endorse or promote products derived |
| 35 | * from this software without specific prior written permission. |
| 36 | * |
| 37 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 38 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 39 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 40 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 41 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 42 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 43 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 44 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 45 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 46 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 47 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 48 | * |
| 49 | */ |
| 50 | |
| 51 | #include <linux/delay.h> |
| 52 | #include "hfi.h" |
| 53 | #include "qp.h" |
| 54 | #include "trace.h" |
| 55 | |
| 56 | #define SC_CTXT_PACKET_EGRESS_TIMEOUT 350 /* in chip cycles */ |
| 57 | |
| 58 | #define SC(name) SEND_CTXT_##name |
| 59 | /* |
| 60 | * Send Context functions |
| 61 | */ |
| 62 | static void sc_wait_for_packet_egress(struct send_context *sc, int pause); |
| 63 | |
| 64 | /* |
| 65 | * Set the CM reset bit and wait for it to clear. Use the provided |
| 66 | * sendctrl register. This routine has no locking. |
| 67 | */ |
| 68 | void __cm_reset(struct hfi1_devdata *dd, u64 sendctrl) |
| 69 | { |
| 70 | write_csr(dd, SEND_CTRL, sendctrl | SEND_CTRL_CM_RESET_SMASK); |
| 71 | while (1) { |
| 72 | udelay(1); |
| 73 | sendctrl = read_csr(dd, SEND_CTRL); |
| 74 | if ((sendctrl & SEND_CTRL_CM_RESET_SMASK) == 0) |
| 75 | break; |
| 76 | } |
| 77 | } |
| 78 | |
| 79 | /* defined in header release 48 and higher */ |
| 80 | #ifndef SEND_CTRL_UNSUPPORTED_VL_SHIFT |
| 81 | #define SEND_CTRL_UNSUPPORTED_VL_SHIFT 3 |
| 82 | #define SEND_CTRL_UNSUPPORTED_VL_MASK 0xffull |
| 83 | #define SEND_CTRL_UNSUPPORTED_VL_SMASK (SEND_CTRL_UNSUPPORTED_VL_MASK \ |
| 84 | << SEND_CTRL_UNSUPPORTED_VL_SHIFT) |
| 85 | #endif |
| 86 | |
| 87 | /* global control of PIO send */ |
| 88 | void pio_send_control(struct hfi1_devdata *dd, int op) |
| 89 | { |
| 90 | u64 reg, mask; |
| 91 | unsigned long flags; |
| 92 | int write = 1; /* write sendctrl back */ |
| 93 | int flush = 0; /* re-read sendctrl to make sure it is flushed */ |
| 94 | |
| 95 | spin_lock_irqsave(&dd->sendctrl_lock, flags); |
| 96 | |
| 97 | reg = read_csr(dd, SEND_CTRL); |
| 98 | switch (op) { |
| 99 | case PSC_GLOBAL_ENABLE: |
| 100 | reg |= SEND_CTRL_SEND_ENABLE_SMASK; |
| 101 | /* Fall through */ |
| 102 | case PSC_DATA_VL_ENABLE: |
| 103 | /* Disallow sending on VLs not enabled */ |
| 104 | mask = (((~0ull)<<num_vls) & SEND_CTRL_UNSUPPORTED_VL_MASK)<< |
| 105 | SEND_CTRL_UNSUPPORTED_VL_SHIFT; |
| 106 | reg = (reg & ~SEND_CTRL_UNSUPPORTED_VL_SMASK) | mask; |
| 107 | break; |
| 108 | case PSC_GLOBAL_DISABLE: |
| 109 | reg &= ~SEND_CTRL_SEND_ENABLE_SMASK; |
| 110 | break; |
| 111 | case PSC_GLOBAL_VLARB_ENABLE: |
| 112 | reg |= SEND_CTRL_VL_ARBITER_ENABLE_SMASK; |
| 113 | break; |
| 114 | case PSC_GLOBAL_VLARB_DISABLE: |
| 115 | reg &= ~SEND_CTRL_VL_ARBITER_ENABLE_SMASK; |
| 116 | break; |
| 117 | case PSC_CM_RESET: |
| 118 | __cm_reset(dd, reg); |
| 119 | write = 0; /* CSR already written (and flushed) */ |
| 120 | break; |
| 121 | case PSC_DATA_VL_DISABLE: |
| 122 | reg |= SEND_CTRL_UNSUPPORTED_VL_SMASK; |
| 123 | flush = 1; |
| 124 | break; |
| 125 | default: |
| 126 | dd_dev_err(dd, "%s: invalid control %d\n", __func__, op); |
| 127 | break; |
| 128 | } |
| 129 | |
| 130 | if (write) { |
| 131 | write_csr(dd, SEND_CTRL, reg); |
| 132 | if (flush) |
| 133 | (void) read_csr(dd, SEND_CTRL); /* flush write */ |
| 134 | } |
| 135 | |
| 136 | spin_unlock_irqrestore(&dd->sendctrl_lock, flags); |
| 137 | } |
| 138 | |
| 139 | /* number of send context memory pools */ |
| 140 | #define NUM_SC_POOLS 2 |
| 141 | |
| 142 | /* Send Context Size (SCS) wildcards */ |
| 143 | #define SCS_POOL_0 -1 |
| 144 | #define SCS_POOL_1 -2 |
| 145 | /* Send Context Count (SCC) wildcards */ |
| 146 | #define SCC_PER_VL -1 |
| 147 | #define SCC_PER_CPU -2 |
| 148 | |
| 149 | #define SCC_PER_KRCVQ -3 |
| 150 | #define SCC_ACK_CREDITS 32 |
| 151 | |
| 152 | #define PIO_WAIT_BATCH_SIZE 5 |
| 153 | |
| 154 | /* default send context sizes */ |
| 155 | static struct sc_config_sizes sc_config_sizes[SC_MAX] = { |
| 156 | [SC_KERNEL] = { .size = SCS_POOL_0, /* even divide, pool 0 */ |
| 157 | .count = SCC_PER_VL },/* one per NUMA */ |
| 158 | [SC_ACK] = { .size = SCC_ACK_CREDITS, |
| 159 | .count = SCC_PER_KRCVQ }, |
| 160 | [SC_USER] = { .size = SCS_POOL_0, /* even divide, pool 0 */ |
| 161 | .count = SCC_PER_CPU }, /* one per CPU */ |
| 162 | |
| 163 | }; |
| 164 | |
| 165 | /* send context memory pool configuration */ |
| 166 | struct mem_pool_config { |
| 167 | int centipercent; /* % of memory, in 100ths of 1% */ |
| 168 | int absolute_blocks; /* absolute block count */ |
| 169 | }; |
| 170 | |
| 171 | /* default memory pool configuration: 100% in pool 0 */ |
| 172 | static struct mem_pool_config sc_mem_pool_config[NUM_SC_POOLS] = { |
| 173 | /* centi%, abs blocks */ |
| 174 | { 10000, -1 }, /* pool 0 */ |
| 175 | { 0, -1 }, /* pool 1 */ |
| 176 | }; |
| 177 | |
| 178 | /* memory pool information, used when calculating final sizes */ |
| 179 | struct mem_pool_info { |
| 180 | int centipercent; /* 100th of 1% of memory to use, -1 if blocks |
| 181 | already set */ |
| 182 | int count; /* count of contexts in the pool */ |
| 183 | int blocks; /* block size of the pool */ |
| 184 | int size; /* context size, in blocks */ |
| 185 | }; |
| 186 | |
| 187 | /* |
| 188 | * Convert a pool wildcard to a valid pool index. The wildcards |
| 189 | * start at -1 and increase negatively. Map them as: |
| 190 | * -1 => 0 |
| 191 | * -2 => 1 |
| 192 | * etc. |
| 193 | * |
| 194 | * Return -1 on non-wildcard input, otherwise convert to a pool number. |
| 195 | */ |
| 196 | static int wildcard_to_pool(int wc) |
| 197 | { |
| 198 | if (wc >= 0) |
| 199 | return -1; /* non-wildcard */ |
| 200 | return -wc - 1; |
| 201 | } |
| 202 | |
| 203 | static const char *sc_type_names[SC_MAX] = { |
| 204 | "kernel", |
| 205 | "ack", |
| 206 | "user" |
| 207 | }; |
| 208 | |
| 209 | static const char *sc_type_name(int index) |
| 210 | { |
| 211 | if (index < 0 || index >= SC_MAX) |
| 212 | return "unknown"; |
| 213 | return sc_type_names[index]; |
| 214 | } |
| 215 | |
| 216 | /* |
| 217 | * Read the send context memory pool configuration and send context |
| 218 | * size configuration. Replace any wildcards and come up with final |
| 219 | * counts and sizes for the send context types. |
| 220 | */ |
| 221 | int init_sc_pools_and_sizes(struct hfi1_devdata *dd) |
| 222 | { |
| 223 | struct mem_pool_info mem_pool_info[NUM_SC_POOLS] = { { 0 } }; |
| 224 | int total_blocks = (dd->chip_pio_mem_size / PIO_BLOCK_SIZE) - 1; |
| 225 | int total_contexts = 0; |
| 226 | int fixed_blocks; |
| 227 | int pool_blocks; |
| 228 | int used_blocks; |
| 229 | int cp_total; /* centipercent total */ |
| 230 | int ab_total; /* absolute block total */ |
| 231 | int extra; |
| 232 | int i; |
| 233 | |
| 234 | /* |
| 235 | * Step 0: |
| 236 | * - copy the centipercents/absolute sizes from the pool config |
| 237 | * - sanity check these values |
| 238 | * - add up centipercents, then later check for full value |
| 239 | * - add up absolute blocks, then later check for over-commit |
| 240 | */ |
| 241 | cp_total = 0; |
| 242 | ab_total = 0; |
| 243 | for (i = 0; i < NUM_SC_POOLS; i++) { |
| 244 | int cp = sc_mem_pool_config[i].centipercent; |
| 245 | int ab = sc_mem_pool_config[i].absolute_blocks; |
| 246 | |
| 247 | /* |
| 248 | * A negative value is "unused" or "invalid". Both *can* |
| 249 | * be valid, but centipercent wins, so check that first |
| 250 | */ |
| 251 | if (cp >= 0) { /* centipercent valid */ |
| 252 | cp_total += cp; |
| 253 | } else if (ab >= 0) { /* absolute blocks valid */ |
| 254 | ab_total += ab; |
| 255 | } else { /* neither valid */ |
| 256 | dd_dev_err( |
| 257 | dd, |
| 258 | "Send context memory pool %d: both the block count and centipercent are invalid\n", |
| 259 | i); |
| 260 | return -EINVAL; |
| 261 | } |
| 262 | |
| 263 | mem_pool_info[i].centipercent = cp; |
| 264 | mem_pool_info[i].blocks = ab; |
| 265 | } |
| 266 | |
| 267 | /* do not use both % and absolute blocks for different pools */ |
| 268 | if (cp_total != 0 && ab_total != 0) { |
| 269 | dd_dev_err( |
| 270 | dd, |
| 271 | "All send context memory pools must be described as either centipercent or blocks, no mixing between pools\n"); |
| 272 | return -EINVAL; |
| 273 | } |
| 274 | |
| 275 | /* if any percentages are present, they must add up to 100% x 100 */ |
| 276 | if (cp_total != 0 && cp_total != 10000) { |
| 277 | dd_dev_err( |
| 278 | dd, |
| 279 | "Send context memory pool centipercent is %d, expecting 10000\n", |
| 280 | cp_total); |
| 281 | return -EINVAL; |
| 282 | } |
| 283 | |
| 284 | /* the absolute pool total cannot be more than the mem total */ |
| 285 | if (ab_total > total_blocks) { |
| 286 | dd_dev_err( |
| 287 | dd, |
| 288 | "Send context memory pool absolute block count %d is larger than the memory size %d\n", |
| 289 | ab_total, total_blocks); |
| 290 | return -EINVAL; |
| 291 | } |
| 292 | |
| 293 | /* |
| 294 | * Step 2: |
| 295 | * - copy from the context size config |
| 296 | * - replace context type wildcard counts with real values |
| 297 | * - add up non-memory pool block sizes |
| 298 | * - add up memory pool user counts |
| 299 | */ |
| 300 | fixed_blocks = 0; |
| 301 | for (i = 0; i < SC_MAX; i++) { |
| 302 | int count = sc_config_sizes[i].count; |
| 303 | int size = sc_config_sizes[i].size; |
| 304 | int pool; |
| 305 | |
| 306 | /* |
| 307 | * Sanity check count: Either a positive value or |
| 308 | * one of the expected wildcards is valid. The positive |
| 309 | * value is checked later when we compare against total |
| 310 | * memory available. |
| 311 | */ |
| 312 | if (i == SC_ACK) { |
| 313 | count = dd->n_krcv_queues; |
| 314 | } else if (i == SC_KERNEL) { |
| 315 | count = num_vls + 1 /* VL15 */; |
| 316 | } else if (count == SCC_PER_CPU) { |
| 317 | count = dd->num_rcv_contexts - dd->n_krcv_queues; |
| 318 | } else if (count < 0) { |
| 319 | dd_dev_err( |
| 320 | dd, |
| 321 | "%s send context invalid count wildcard %d\n", |
| 322 | sc_type_name(i), count); |
| 323 | return -EINVAL; |
| 324 | } |
| 325 | if (total_contexts + count > dd->chip_send_contexts) |
| 326 | count = dd->chip_send_contexts - total_contexts; |
| 327 | |
| 328 | total_contexts += count; |
| 329 | |
| 330 | /* |
| 331 | * Sanity check pool: The conversion will return a pool |
| 332 | * number or -1 if a fixed (non-negative) value. The fixed |
| 333 | * value is checked later when we compare against |
| 334 | * total memory available. |
| 335 | */ |
| 336 | pool = wildcard_to_pool(size); |
| 337 | if (pool == -1) { /* non-wildcard */ |
| 338 | fixed_blocks += size * count; |
| 339 | } else if (pool < NUM_SC_POOLS) { /* valid wildcard */ |
| 340 | mem_pool_info[pool].count += count; |
| 341 | } else { /* invalid wildcard */ |
| 342 | dd_dev_err( |
| 343 | dd, |
| 344 | "%s send context invalid pool wildcard %d\n", |
| 345 | sc_type_name(i), size); |
| 346 | return -EINVAL; |
| 347 | } |
| 348 | |
| 349 | dd->sc_sizes[i].count = count; |
| 350 | dd->sc_sizes[i].size = size; |
| 351 | } |
| 352 | if (fixed_blocks > total_blocks) { |
| 353 | dd_dev_err( |
| 354 | dd, |
| 355 | "Send context fixed block count, %u, larger than total block count %u\n", |
| 356 | fixed_blocks, total_blocks); |
| 357 | return -EINVAL; |
| 358 | } |
| 359 | |
| 360 | /* step 3: calculate the blocks in the pools, and pool context sizes */ |
| 361 | pool_blocks = total_blocks - fixed_blocks; |
| 362 | if (ab_total > pool_blocks) { |
| 363 | dd_dev_err( |
| 364 | dd, |
| 365 | "Send context fixed pool sizes, %u, larger than pool block count %u\n", |
| 366 | ab_total, pool_blocks); |
| 367 | return -EINVAL; |
| 368 | } |
| 369 | /* subtract off the fixed pool blocks */ |
| 370 | pool_blocks -= ab_total; |
| 371 | |
| 372 | for (i = 0; i < NUM_SC_POOLS; i++) { |
| 373 | struct mem_pool_info *pi = &mem_pool_info[i]; |
| 374 | |
| 375 | /* % beats absolute blocks */ |
| 376 | if (pi->centipercent >= 0) |
| 377 | pi->blocks = (pool_blocks * pi->centipercent) / 10000; |
| 378 | |
| 379 | if (pi->blocks == 0 && pi->count != 0) { |
| 380 | dd_dev_err( |
| 381 | dd, |
| 382 | "Send context memory pool %d has %u contexts, but no blocks\n", |
| 383 | i, pi->count); |
| 384 | return -EINVAL; |
| 385 | } |
| 386 | if (pi->count == 0) { |
| 387 | /* warn about wasted blocks */ |
| 388 | if (pi->blocks != 0) |
| 389 | dd_dev_err( |
| 390 | dd, |
| 391 | "Send context memory pool %d has %u blocks, but zero contexts\n", |
| 392 | i, pi->blocks); |
| 393 | pi->size = 0; |
| 394 | } else { |
| 395 | pi->size = pi->blocks / pi->count; |
| 396 | } |
| 397 | } |
| 398 | |
| 399 | /* step 4: fill in the context type sizes from the pool sizes */ |
| 400 | used_blocks = 0; |
| 401 | for (i = 0; i < SC_MAX; i++) { |
| 402 | if (dd->sc_sizes[i].size < 0) { |
| 403 | unsigned pool = wildcard_to_pool(dd->sc_sizes[i].size); |
| 404 | |
| 405 | WARN_ON_ONCE(pool >= NUM_SC_POOLS); |
| 406 | dd->sc_sizes[i].size = mem_pool_info[pool].size; |
| 407 | } |
| 408 | /* make sure we are not larger than what is allowed by the HW */ |
| 409 | #define PIO_MAX_BLOCKS 1024 |
| 410 | if (dd->sc_sizes[i].size > PIO_MAX_BLOCKS) |
| 411 | dd->sc_sizes[i].size = PIO_MAX_BLOCKS; |
| 412 | |
| 413 | /* calculate our total usage */ |
| 414 | used_blocks += dd->sc_sizes[i].size * dd->sc_sizes[i].count; |
| 415 | } |
| 416 | extra = total_blocks - used_blocks; |
| 417 | if (extra != 0) |
| 418 | dd_dev_info(dd, "unused send context blocks: %d\n", extra); |
| 419 | |
| 420 | return total_contexts; |
| 421 | } |
| 422 | |
| 423 | int init_send_contexts(struct hfi1_devdata *dd) |
| 424 | { |
| 425 | u16 base; |
| 426 | int ret, i, j, context; |
| 427 | |
| 428 | ret = init_credit_return(dd); |
| 429 | if (ret) |
| 430 | return ret; |
| 431 | |
| 432 | dd->hw_to_sw = kmalloc_array(TXE_NUM_CONTEXTS, sizeof(u8), |
| 433 | GFP_KERNEL); |
| 434 | dd->send_contexts = kcalloc(dd->num_send_contexts, |
| 435 | sizeof(struct send_context_info), |
| 436 | GFP_KERNEL); |
| 437 | if (!dd->send_contexts || !dd->hw_to_sw) { |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 438 | kfree(dd->hw_to_sw); |
| 439 | kfree(dd->send_contexts); |
| 440 | free_credit_return(dd); |
| 441 | return -ENOMEM; |
| 442 | } |
| 443 | |
| 444 | /* hardware context map starts with invalid send context indices */ |
| 445 | for (i = 0; i < TXE_NUM_CONTEXTS; i++) |
| 446 | dd->hw_to_sw[i] = INVALID_SCI; |
| 447 | |
| 448 | /* |
| 449 | * All send contexts have their credit sizes. Allocate credits |
| 450 | * for each context one after another from the global space. |
| 451 | */ |
| 452 | context = 0; |
| 453 | base = 1; |
| 454 | for (i = 0; i < SC_MAX; i++) { |
| 455 | struct sc_config_sizes *scs = &dd->sc_sizes[i]; |
| 456 | |
| 457 | for (j = 0; j < scs->count; j++) { |
| 458 | struct send_context_info *sci = |
| 459 | &dd->send_contexts[context]; |
| 460 | sci->type = i; |
| 461 | sci->base = base; |
| 462 | sci->credits = scs->size; |
| 463 | |
| 464 | context++; |
| 465 | base += scs->size; |
| 466 | } |
| 467 | } |
| 468 | |
| 469 | return 0; |
| 470 | } |
| 471 | |
| 472 | /* |
| 473 | * Allocate a software index and hardware context of the given type. |
| 474 | * |
| 475 | * Must be called with dd->sc_lock held. |
| 476 | */ |
| 477 | static int sc_hw_alloc(struct hfi1_devdata *dd, int type, u32 *sw_index, |
| 478 | u32 *hw_context) |
| 479 | { |
| 480 | struct send_context_info *sci; |
| 481 | u32 index; |
| 482 | u32 context; |
| 483 | |
| 484 | for (index = 0, sci = &dd->send_contexts[0]; |
| 485 | index < dd->num_send_contexts; index++, sci++) { |
| 486 | if (sci->type == type && sci->allocated == 0) { |
| 487 | sci->allocated = 1; |
| 488 | /* use a 1:1 mapping, but make them non-equal */ |
| 489 | context = dd->chip_send_contexts - index - 1; |
| 490 | dd->hw_to_sw[context] = index; |
| 491 | *sw_index = index; |
| 492 | *hw_context = context; |
| 493 | return 0; /* success */ |
| 494 | } |
| 495 | } |
| 496 | dd_dev_err(dd, "Unable to locate a free type %d send context\n", type); |
| 497 | return -ENOSPC; |
| 498 | } |
| 499 | |
| 500 | /* |
| 501 | * Free the send context given by its software index. |
| 502 | * |
| 503 | * Must be called with dd->sc_lock held. |
| 504 | */ |
| 505 | static void sc_hw_free(struct hfi1_devdata *dd, u32 sw_index, u32 hw_context) |
| 506 | { |
| 507 | struct send_context_info *sci; |
| 508 | |
| 509 | sci = &dd->send_contexts[sw_index]; |
| 510 | if (!sci->allocated) { |
| 511 | dd_dev_err(dd, "%s: sw_index %u not allocated? hw_context %u\n", |
| 512 | __func__, sw_index, hw_context); |
| 513 | } |
| 514 | sci->allocated = 0; |
| 515 | dd->hw_to_sw[hw_context] = INVALID_SCI; |
| 516 | } |
| 517 | |
| 518 | /* return the base context of a context in a group */ |
| 519 | static inline u32 group_context(u32 context, u32 group) |
| 520 | { |
| 521 | return (context >> group) << group; |
| 522 | } |
| 523 | |
| 524 | /* return the size of a group */ |
| 525 | static inline u32 group_size(u32 group) |
| 526 | { |
| 527 | return 1 << group; |
| 528 | } |
| 529 | |
| 530 | /* |
| 531 | * Obtain the credit return addresses, kernel virtual and physical, for the |
| 532 | * given sc. |
| 533 | * |
| 534 | * To understand this routine: |
| 535 | * o va and pa are arrays of struct credit_return. One for each physical |
| 536 | * send context, per NUMA. |
| 537 | * o Each send context always looks in its relative location in a struct |
| 538 | * credit_return for its credit return. |
| 539 | * o Each send context in a group must have its return address CSR programmed |
| 540 | * with the same value. Use the address of the first send context in the |
| 541 | * group. |
| 542 | */ |
| 543 | static void cr_group_addresses(struct send_context *sc, dma_addr_t *pa) |
| 544 | { |
| 545 | u32 gc = group_context(sc->hw_context, sc->group); |
| 546 | u32 index = sc->hw_context & 0x7; |
| 547 | |
| 548 | sc->hw_free = &sc->dd->cr_base[sc->node].va[gc].cr[index]; |
| 549 | *pa = (unsigned long) |
| 550 | &((struct credit_return *)sc->dd->cr_base[sc->node].pa)[gc]; |
| 551 | } |
| 552 | |
| 553 | /* |
| 554 | * Work queue function triggered in error interrupt routine for |
| 555 | * kernel contexts. |
| 556 | */ |
| 557 | static void sc_halted(struct work_struct *work) |
| 558 | { |
| 559 | struct send_context *sc; |
| 560 | |
| 561 | sc = container_of(work, struct send_context, halt_work); |
| 562 | sc_restart(sc); |
| 563 | } |
| 564 | |
| 565 | /* |
| 566 | * Calculate PIO block threshold for this send context using the given MTU. |
| 567 | * Trigger a return when one MTU plus optional header of credits remain. |
| 568 | * |
| 569 | * Parameter mtu is in bytes. |
| 570 | * Parameter hdrqentsize is in DWORDs. |
| 571 | * |
| 572 | * Return value is what to write into the CSR: trigger return when |
| 573 | * unreturned credits pass this count. |
| 574 | */ |
| 575 | u32 sc_mtu_to_threshold(struct send_context *sc, u32 mtu, u32 hdrqentsize) |
| 576 | { |
| 577 | u32 release_credits; |
| 578 | u32 threshold; |
| 579 | |
| 580 | /* add in the header size, then divide by the PIO block size */ |
| 581 | mtu += hdrqentsize << 2; |
| 582 | release_credits = DIV_ROUND_UP(mtu, PIO_BLOCK_SIZE); |
| 583 | |
| 584 | /* check against this context's credits */ |
| 585 | if (sc->credits <= release_credits) |
| 586 | threshold = 1; |
| 587 | else |
| 588 | threshold = sc->credits - release_credits; |
| 589 | |
| 590 | return threshold; |
| 591 | } |
| 592 | |
| 593 | /* |
| 594 | * Calculate credit threshold in terms of percent of the allocated credits. |
| 595 | * Trigger when unreturned credits equal or exceed the percentage of the whole. |
| 596 | * |
| 597 | * Return value is what to write into the CSR: trigger return when |
| 598 | * unreturned credits pass this count. |
| 599 | */ |
| 600 | static u32 sc_percent_to_threshold(struct send_context *sc, u32 percent) |
| 601 | { |
| 602 | return (sc->credits * percent) / 100; |
| 603 | } |
| 604 | |
| 605 | /* |
| 606 | * Set the credit return threshold. |
| 607 | */ |
| 608 | void sc_set_cr_threshold(struct send_context *sc, u32 new_threshold) |
| 609 | { |
| 610 | unsigned long flags; |
| 611 | u32 old_threshold; |
| 612 | int force_return = 0; |
| 613 | |
| 614 | spin_lock_irqsave(&sc->credit_ctrl_lock, flags); |
| 615 | |
| 616 | old_threshold = (sc->credit_ctrl >> |
| 617 | SC(CREDIT_CTRL_THRESHOLD_SHIFT)) |
| 618 | & SC(CREDIT_CTRL_THRESHOLD_MASK); |
| 619 | |
| 620 | if (new_threshold != old_threshold) { |
| 621 | sc->credit_ctrl = |
| 622 | (sc->credit_ctrl |
| 623 | & ~SC(CREDIT_CTRL_THRESHOLD_SMASK)) |
| 624 | | ((new_threshold |
| 625 | & SC(CREDIT_CTRL_THRESHOLD_MASK)) |
| 626 | << SC(CREDIT_CTRL_THRESHOLD_SHIFT)); |
| 627 | write_kctxt_csr(sc->dd, sc->hw_context, |
| 628 | SC(CREDIT_CTRL), sc->credit_ctrl); |
| 629 | |
| 630 | /* force a credit return on change to avoid a possible stall */ |
| 631 | force_return = 1; |
| 632 | } |
| 633 | |
| 634 | spin_unlock_irqrestore(&sc->credit_ctrl_lock, flags); |
| 635 | |
| 636 | if (force_return) |
| 637 | sc_return_credits(sc); |
| 638 | } |
| 639 | |
| 640 | /* |
| 641 | * set_pio_integrity |
| 642 | * |
| 643 | * Set the CHECK_ENABLE register for the send context 'sc'. |
| 644 | */ |
| 645 | void set_pio_integrity(struct send_context *sc) |
| 646 | { |
| 647 | struct hfi1_devdata *dd = sc->dd; |
| 648 | u64 reg = 0; |
| 649 | u32 hw_context = sc->hw_context; |
| 650 | int type = sc->type; |
| 651 | |
| 652 | /* |
| 653 | * No integrity checks if HFI1_CAP_NO_INTEGRITY is set, or if |
| 654 | * we're snooping. |
| 655 | */ |
| 656 | if (likely(!HFI1_CAP_IS_KSET(NO_INTEGRITY)) && |
| 657 | dd->hfi1_snoop.mode_flag != HFI1_PORT_SNOOP_MODE) |
| 658 | reg = hfi1_pkt_default_send_ctxt_mask(dd, type); |
| 659 | |
| 660 | write_kctxt_csr(dd, hw_context, SC(CHECK_ENABLE), reg); |
| 661 | } |
| 662 | |
Mike Marciniszyn | a054374 | 2015-12-07 15:39:22 -0500 | [diff] [blame] | 663 | static u32 get_buffers_allocated(struct send_context *sc) |
| 664 | { |
| 665 | int cpu; |
| 666 | u32 ret = 0; |
| 667 | |
| 668 | for_each_possible_cpu(cpu) |
| 669 | ret += *per_cpu_ptr(sc->buffers_allocated, cpu); |
| 670 | return ret; |
| 671 | } |
| 672 | |
| 673 | static void reset_buffers_allocated(struct send_context *sc) |
| 674 | { |
| 675 | int cpu; |
| 676 | |
| 677 | for_each_possible_cpu(cpu) |
| 678 | (*per_cpu_ptr(sc->buffers_allocated, cpu)) = 0; |
| 679 | } |
| 680 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 681 | /* |
| 682 | * Allocate a NUMA relative send context structure of the given type along |
| 683 | * with a HW context. |
| 684 | */ |
| 685 | struct send_context *sc_alloc(struct hfi1_devdata *dd, int type, |
| 686 | uint hdrqentsize, int numa) |
| 687 | { |
| 688 | struct send_context_info *sci; |
Mike Marciniszyn | a054374 | 2015-12-07 15:39:22 -0500 | [diff] [blame] | 689 | struct send_context *sc = NULL; |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 690 | dma_addr_t pa; |
| 691 | unsigned long flags; |
| 692 | u64 reg; |
| 693 | u32 thresh; |
| 694 | u32 sw_index; |
| 695 | u32 hw_context; |
| 696 | int ret; |
| 697 | u8 opval, opmask; |
| 698 | |
| 699 | /* do not allocate while frozen */ |
| 700 | if (dd->flags & HFI1_FROZEN) |
| 701 | return NULL; |
| 702 | |
| 703 | sc = kzalloc_node(sizeof(struct send_context), GFP_KERNEL, numa); |
Alison Schofield | 806e6e1 | 2015-10-12 14:28:36 -0700 | [diff] [blame] | 704 | if (!sc) |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 705 | return NULL; |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 706 | |
Mike Marciniszyn | a054374 | 2015-12-07 15:39:22 -0500 | [diff] [blame] | 707 | sc->buffers_allocated = alloc_percpu(u32); |
| 708 | if (!sc->buffers_allocated) { |
| 709 | kfree(sc); |
| 710 | dd_dev_err(dd, |
| 711 | "Cannot allocate buffers_allocated per cpu counters\n" |
| 712 | ); |
| 713 | return NULL; |
| 714 | } |
| 715 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 716 | spin_lock_irqsave(&dd->sc_lock, flags); |
| 717 | ret = sc_hw_alloc(dd, type, &sw_index, &hw_context); |
| 718 | if (ret) { |
| 719 | spin_unlock_irqrestore(&dd->sc_lock, flags); |
Mike Marciniszyn | a054374 | 2015-12-07 15:39:22 -0500 | [diff] [blame] | 720 | free_percpu(sc->buffers_allocated); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 721 | kfree(sc); |
| 722 | return NULL; |
| 723 | } |
| 724 | |
| 725 | sci = &dd->send_contexts[sw_index]; |
| 726 | sci->sc = sc; |
| 727 | |
| 728 | sc->dd = dd; |
| 729 | sc->node = numa; |
| 730 | sc->type = type; |
| 731 | spin_lock_init(&sc->alloc_lock); |
| 732 | spin_lock_init(&sc->release_lock); |
| 733 | spin_lock_init(&sc->credit_ctrl_lock); |
| 734 | INIT_LIST_HEAD(&sc->piowait); |
| 735 | INIT_WORK(&sc->halt_work, sc_halted); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 736 | init_waitqueue_head(&sc->halt_wait); |
| 737 | |
| 738 | /* grouping is always single context for now */ |
| 739 | sc->group = 0; |
| 740 | |
| 741 | sc->sw_index = sw_index; |
| 742 | sc->hw_context = hw_context; |
| 743 | cr_group_addresses(sc, &pa); |
| 744 | sc->credits = sci->credits; |
| 745 | |
| 746 | /* PIO Send Memory Address details */ |
| 747 | #define PIO_ADDR_CONTEXT_MASK 0xfful |
| 748 | #define PIO_ADDR_CONTEXT_SHIFT 16 |
| 749 | sc->base_addr = dd->piobase + ((hw_context & PIO_ADDR_CONTEXT_MASK) |
| 750 | << PIO_ADDR_CONTEXT_SHIFT); |
| 751 | |
| 752 | /* set base and credits */ |
| 753 | reg = ((sci->credits & SC(CTRL_CTXT_DEPTH_MASK)) |
| 754 | << SC(CTRL_CTXT_DEPTH_SHIFT)) |
| 755 | | ((sci->base & SC(CTRL_CTXT_BASE_MASK)) |
| 756 | << SC(CTRL_CTXT_BASE_SHIFT)); |
| 757 | write_kctxt_csr(dd, hw_context, SC(CTRL), reg); |
| 758 | |
| 759 | set_pio_integrity(sc); |
| 760 | |
| 761 | /* unmask all errors */ |
| 762 | write_kctxt_csr(dd, hw_context, SC(ERR_MASK), (u64)-1); |
| 763 | |
| 764 | /* set the default partition key */ |
| 765 | write_kctxt_csr(dd, hw_context, SC(CHECK_PARTITION_KEY), |
| 766 | (DEFAULT_PKEY & |
| 767 | SC(CHECK_PARTITION_KEY_VALUE_MASK)) |
| 768 | << SC(CHECK_PARTITION_KEY_VALUE_SHIFT)); |
| 769 | |
| 770 | /* per context type checks */ |
| 771 | if (type == SC_USER) { |
| 772 | opval = USER_OPCODE_CHECK_VAL; |
| 773 | opmask = USER_OPCODE_CHECK_MASK; |
| 774 | } else { |
| 775 | opval = OPCODE_CHECK_VAL_DISABLED; |
| 776 | opmask = OPCODE_CHECK_MASK_DISABLED; |
| 777 | } |
| 778 | |
| 779 | /* set the send context check opcode mask and value */ |
| 780 | write_kctxt_csr(dd, hw_context, SC(CHECK_OPCODE), |
| 781 | ((u64)opmask << SC(CHECK_OPCODE_MASK_SHIFT)) | |
| 782 | ((u64)opval << SC(CHECK_OPCODE_VALUE_SHIFT))); |
| 783 | |
| 784 | /* set up credit return */ |
| 785 | reg = pa & SC(CREDIT_RETURN_ADDR_ADDRESS_SMASK); |
| 786 | write_kctxt_csr(dd, hw_context, SC(CREDIT_RETURN_ADDR), reg); |
| 787 | |
| 788 | /* |
| 789 | * Calculate the initial credit return threshold. |
| 790 | * |
| 791 | * For Ack contexts, set a threshold for half the credits. |
| 792 | * For User contexts use the given percentage. This has been |
| 793 | * sanitized on driver start-up. |
| 794 | * For Kernel contexts, use the default MTU plus a header. |
| 795 | */ |
| 796 | if (type == SC_ACK) { |
| 797 | thresh = sc_percent_to_threshold(sc, 50); |
| 798 | } else if (type == SC_USER) { |
| 799 | thresh = sc_percent_to_threshold(sc, |
| 800 | user_credit_return_threshold); |
| 801 | } else { /* kernel */ |
| 802 | thresh = sc_mtu_to_threshold(sc, hfi1_max_mtu, hdrqentsize); |
| 803 | } |
| 804 | reg = thresh << SC(CREDIT_CTRL_THRESHOLD_SHIFT); |
| 805 | /* add in early return */ |
| 806 | if (type == SC_USER && HFI1_CAP_IS_USET(EARLY_CREDIT_RETURN)) |
| 807 | reg |= SC(CREDIT_CTRL_EARLY_RETURN_SMASK); |
| 808 | else if (HFI1_CAP_IS_KSET(EARLY_CREDIT_RETURN)) /* kernel, ack */ |
| 809 | reg |= SC(CREDIT_CTRL_EARLY_RETURN_SMASK); |
| 810 | |
| 811 | /* set up write-through credit_ctrl */ |
| 812 | sc->credit_ctrl = reg; |
| 813 | write_kctxt_csr(dd, hw_context, SC(CREDIT_CTRL), reg); |
| 814 | |
| 815 | /* User send contexts should not allow sending on VL15 */ |
| 816 | if (type == SC_USER) { |
| 817 | reg = 1ULL << 15; |
| 818 | write_kctxt_csr(dd, hw_context, SC(CHECK_VL), reg); |
| 819 | } |
| 820 | |
| 821 | spin_unlock_irqrestore(&dd->sc_lock, flags); |
| 822 | |
| 823 | /* |
| 824 | * Allocate shadow ring to track outstanding PIO buffers _after_ |
| 825 | * unlocking. We don't know the size until the lock is held and |
| 826 | * we can't allocate while the lock is held. No one is using |
| 827 | * the context yet, so allocate it now. |
| 828 | * |
| 829 | * User contexts do not get a shadow ring. |
| 830 | */ |
| 831 | if (type != SC_USER) { |
| 832 | /* |
| 833 | * Size the shadow ring 1 larger than the number of credits |
| 834 | * so head == tail can mean empty. |
| 835 | */ |
| 836 | sc->sr_size = sci->credits + 1; |
| 837 | sc->sr = kzalloc_node(sizeof(union pio_shadow_ring) * |
| 838 | sc->sr_size, GFP_KERNEL, numa); |
| 839 | if (!sc->sr) { |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 840 | sc_free(sc); |
| 841 | return NULL; |
| 842 | } |
| 843 | } |
| 844 | |
Sebastian Sanchez | 6c63e42 | 2015-11-06 20:06:56 -0500 | [diff] [blame] | 845 | hfi1_cdbg(PIO, |
| 846 | "Send context %u(%u) %s group %u credits %u credit_ctrl 0x%llx threshold %u\n", |
| 847 | sw_index, |
| 848 | hw_context, |
| 849 | sc_type_name(type), |
| 850 | sc->group, |
| 851 | sc->credits, |
| 852 | sc->credit_ctrl, |
| 853 | thresh); |
| 854 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 855 | |
| 856 | return sc; |
| 857 | } |
| 858 | |
| 859 | /* free a per-NUMA send context structure */ |
| 860 | void sc_free(struct send_context *sc) |
| 861 | { |
| 862 | struct hfi1_devdata *dd; |
| 863 | unsigned long flags; |
| 864 | u32 sw_index; |
| 865 | u32 hw_context; |
| 866 | |
| 867 | if (!sc) |
| 868 | return; |
| 869 | |
| 870 | sc->flags |= SCF_IN_FREE; /* ensure no restarts */ |
| 871 | dd = sc->dd; |
| 872 | if (!list_empty(&sc->piowait)) |
| 873 | dd_dev_err(dd, "piowait list not empty!\n"); |
| 874 | sw_index = sc->sw_index; |
| 875 | hw_context = sc->hw_context; |
| 876 | sc_disable(sc); /* make sure the HW is disabled */ |
| 877 | flush_work(&sc->halt_work); |
| 878 | |
| 879 | spin_lock_irqsave(&dd->sc_lock, flags); |
| 880 | dd->send_contexts[sw_index].sc = NULL; |
| 881 | |
| 882 | /* clear/disable all registers set in sc_alloc */ |
| 883 | write_kctxt_csr(dd, hw_context, SC(CTRL), 0); |
| 884 | write_kctxt_csr(dd, hw_context, SC(CHECK_ENABLE), 0); |
| 885 | write_kctxt_csr(dd, hw_context, SC(ERR_MASK), 0); |
| 886 | write_kctxt_csr(dd, hw_context, SC(CHECK_PARTITION_KEY), 0); |
| 887 | write_kctxt_csr(dd, hw_context, SC(CHECK_OPCODE), 0); |
| 888 | write_kctxt_csr(dd, hw_context, SC(CREDIT_RETURN_ADDR), 0); |
| 889 | write_kctxt_csr(dd, hw_context, SC(CREDIT_CTRL), 0); |
| 890 | |
| 891 | /* release the index and context for re-use */ |
| 892 | sc_hw_free(dd, sw_index, hw_context); |
| 893 | spin_unlock_irqrestore(&dd->sc_lock, flags); |
| 894 | |
| 895 | kfree(sc->sr); |
Mike Marciniszyn | a054374 | 2015-12-07 15:39:22 -0500 | [diff] [blame] | 896 | free_percpu(sc->buffers_allocated); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 897 | kfree(sc); |
| 898 | } |
| 899 | |
| 900 | /* disable the context */ |
| 901 | void sc_disable(struct send_context *sc) |
| 902 | { |
| 903 | u64 reg; |
| 904 | unsigned long flags; |
| 905 | struct pio_buf *pbuf; |
| 906 | |
| 907 | if (!sc) |
| 908 | return; |
| 909 | |
| 910 | /* do all steps, even if already disabled */ |
| 911 | spin_lock_irqsave(&sc->alloc_lock, flags); |
| 912 | reg = read_kctxt_csr(sc->dd, sc->hw_context, SC(CTRL)); |
| 913 | reg &= ~SC(CTRL_CTXT_ENABLE_SMASK); |
| 914 | sc->flags &= ~SCF_ENABLED; |
| 915 | sc_wait_for_packet_egress(sc, 1); |
| 916 | write_kctxt_csr(sc->dd, sc->hw_context, SC(CTRL), reg); |
| 917 | spin_unlock_irqrestore(&sc->alloc_lock, flags); |
| 918 | |
| 919 | /* |
| 920 | * Flush any waiters. Once the context is disabled, |
| 921 | * credit return interrupts are stopped (although there |
| 922 | * could be one in-process when the context is disabled). |
| 923 | * Wait one microsecond for any lingering interrupts, then |
| 924 | * proceed with the flush. |
| 925 | */ |
| 926 | udelay(1); |
| 927 | spin_lock_irqsave(&sc->release_lock, flags); |
| 928 | if (sc->sr) { /* this context has a shadow ring */ |
| 929 | while (sc->sr_tail != sc->sr_head) { |
| 930 | pbuf = &sc->sr[sc->sr_tail].pbuf; |
| 931 | if (pbuf->cb) |
| 932 | (*pbuf->cb)(pbuf->arg, PRC_SC_DISABLE); |
| 933 | sc->sr_tail++; |
| 934 | if (sc->sr_tail >= sc->sr_size) |
| 935 | sc->sr_tail = 0; |
| 936 | } |
| 937 | } |
| 938 | spin_unlock_irqrestore(&sc->release_lock, flags); |
| 939 | } |
| 940 | |
| 941 | /* return SendEgressCtxtStatus.PacketOccupancy */ |
| 942 | #define packet_occupancy(r) \ |
| 943 | (((r) & SEND_EGRESS_CTXT_STATUS_CTXT_EGRESS_PACKET_OCCUPANCY_SMASK)\ |
| 944 | >> SEND_EGRESS_CTXT_STATUS_CTXT_EGRESS_PACKET_OCCUPANCY_SHIFT) |
| 945 | |
| 946 | /* is egress halted on the context? */ |
| 947 | #define egress_halted(r) \ |
| 948 | ((r) & SEND_EGRESS_CTXT_STATUS_CTXT_EGRESS_HALT_STATUS_SMASK) |
| 949 | |
| 950 | /* wait for packet egress, optionally pause for credit return */ |
| 951 | static void sc_wait_for_packet_egress(struct send_context *sc, int pause) |
| 952 | { |
| 953 | struct hfi1_devdata *dd = sc->dd; |
Vennila Megavannan | 25d97dd | 2015-10-26 10:28:30 -0400 | [diff] [blame] | 954 | u64 reg = 0; |
| 955 | u64 reg_prev; |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 956 | u32 loop = 0; |
| 957 | |
| 958 | while (1) { |
Vennila Megavannan | 25d97dd | 2015-10-26 10:28:30 -0400 | [diff] [blame] | 959 | reg_prev = reg; |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 960 | reg = read_csr(dd, sc->hw_context * 8 + |
| 961 | SEND_EGRESS_CTXT_STATUS); |
| 962 | /* done if egress is stopped */ |
| 963 | if (egress_halted(reg)) |
| 964 | break; |
| 965 | reg = packet_occupancy(reg); |
| 966 | if (reg == 0) |
| 967 | break; |
Vennila Megavannan | 25d97dd | 2015-10-26 10:28:30 -0400 | [diff] [blame] | 968 | /* counter is reset if occupancy count changes */ |
| 969 | if (reg != reg_prev) |
| 970 | loop = 0; |
| 971 | if (loop > 500) { |
| 972 | /* timed out - bounce the link */ |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 973 | dd_dev_err(dd, |
Vennila Megavannan | 25d97dd | 2015-10-26 10:28:30 -0400 | [diff] [blame] | 974 | "%s: context %u(%u) timeout waiting for packets to egress, remaining count %u, bouncing link\n", |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 975 | __func__, sc->sw_index, |
| 976 | sc->hw_context, (u32)reg); |
Vennila Megavannan | 25d97dd | 2015-10-26 10:28:30 -0400 | [diff] [blame] | 977 | queue_work(dd->pport->hfi1_wq, |
| 978 | &dd->pport->link_bounce_work); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 979 | break; |
| 980 | } |
| 981 | loop++; |
| 982 | udelay(1); |
| 983 | } |
| 984 | |
| 985 | if (pause) |
| 986 | /* Add additional delay to ensure chip returns all credits */ |
| 987 | pause_for_credit_return(dd); |
| 988 | } |
| 989 | |
| 990 | void sc_wait(struct hfi1_devdata *dd) |
| 991 | { |
| 992 | int i; |
| 993 | |
| 994 | for (i = 0; i < dd->num_send_contexts; i++) { |
| 995 | struct send_context *sc = dd->send_contexts[i].sc; |
| 996 | |
| 997 | if (!sc) |
| 998 | continue; |
| 999 | sc_wait_for_packet_egress(sc, 0); |
| 1000 | } |
| 1001 | } |
| 1002 | |
| 1003 | /* |
| 1004 | * Restart a context after it has been halted due to error. |
| 1005 | * |
| 1006 | * If the first step fails - wait for the halt to be asserted, return early. |
| 1007 | * Otherwise complain about timeouts but keep going. |
| 1008 | * |
| 1009 | * It is expected that allocations (enabled flag bit) have been shut off |
| 1010 | * already (only applies to kernel contexts). |
| 1011 | */ |
| 1012 | int sc_restart(struct send_context *sc) |
| 1013 | { |
| 1014 | struct hfi1_devdata *dd = sc->dd; |
| 1015 | u64 reg; |
| 1016 | u32 loop; |
| 1017 | int count; |
| 1018 | |
| 1019 | /* bounce off if not halted, or being free'd */ |
| 1020 | if (!(sc->flags & SCF_HALTED) || (sc->flags & SCF_IN_FREE)) |
| 1021 | return -EINVAL; |
| 1022 | |
| 1023 | dd_dev_info(dd, "restarting send context %u(%u)\n", sc->sw_index, |
| 1024 | sc->hw_context); |
| 1025 | |
| 1026 | /* |
| 1027 | * Step 1: Wait for the context to actually halt. |
| 1028 | * |
| 1029 | * The error interrupt is asynchronous to actually setting halt |
| 1030 | * on the context. |
| 1031 | */ |
| 1032 | loop = 0; |
| 1033 | while (1) { |
| 1034 | reg = read_kctxt_csr(dd, sc->hw_context, SC(STATUS)); |
| 1035 | if (reg & SC(STATUS_CTXT_HALTED_SMASK)) |
| 1036 | break; |
| 1037 | if (loop > 100) { |
| 1038 | dd_dev_err(dd, "%s: context %u(%u) not halting, skipping\n", |
| 1039 | __func__, sc->sw_index, sc->hw_context); |
| 1040 | return -ETIME; |
| 1041 | } |
| 1042 | loop++; |
| 1043 | udelay(1); |
| 1044 | } |
| 1045 | |
| 1046 | /* |
| 1047 | * Step 2: Ensure no users are still trying to write to PIO. |
| 1048 | * |
| 1049 | * For kernel contexts, we have already turned off buffer allocation. |
| 1050 | * Now wait for the buffer count to go to zero. |
| 1051 | * |
| 1052 | * For user contexts, the user handling code has cut off write access |
| 1053 | * to the context's PIO pages before calling this routine and will |
| 1054 | * restore write access after this routine returns. |
| 1055 | */ |
| 1056 | if (sc->type != SC_USER) { |
| 1057 | /* kernel context */ |
| 1058 | loop = 0; |
| 1059 | while (1) { |
Mike Marciniszyn | a054374 | 2015-12-07 15:39:22 -0500 | [diff] [blame] | 1060 | count = get_buffers_allocated(sc); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 1061 | if (count == 0) |
| 1062 | break; |
| 1063 | if (loop > 100) { |
| 1064 | dd_dev_err(dd, |
| 1065 | "%s: context %u(%u) timeout waiting for PIO buffers to zero, remaining %d\n", |
| 1066 | __func__, sc->sw_index, |
| 1067 | sc->hw_context, count); |
| 1068 | } |
| 1069 | loop++; |
| 1070 | udelay(1); |
| 1071 | } |
| 1072 | } |
| 1073 | |
| 1074 | /* |
| 1075 | * Step 3: Wait for all packets to egress. |
| 1076 | * This is done while disabling the send context |
| 1077 | * |
| 1078 | * Step 4: Disable the context |
| 1079 | * |
| 1080 | * This is a superset of the halt. After the disable, the |
| 1081 | * errors can be cleared. |
| 1082 | */ |
| 1083 | sc_disable(sc); |
| 1084 | |
| 1085 | /* |
| 1086 | * Step 5: Enable the context |
| 1087 | * |
| 1088 | * This enable will clear the halted flag and per-send context |
| 1089 | * error flags. |
| 1090 | */ |
| 1091 | return sc_enable(sc); |
| 1092 | } |
| 1093 | |
| 1094 | /* |
| 1095 | * PIO freeze processing. To be called after the TXE block is fully frozen. |
| 1096 | * Go through all frozen send contexts and disable them. The contexts are |
| 1097 | * already stopped by the freeze. |
| 1098 | */ |
| 1099 | void pio_freeze(struct hfi1_devdata *dd) |
| 1100 | { |
| 1101 | struct send_context *sc; |
| 1102 | int i; |
| 1103 | |
| 1104 | for (i = 0; i < dd->num_send_contexts; i++) { |
| 1105 | sc = dd->send_contexts[i].sc; |
| 1106 | /* |
| 1107 | * Don't disable unallocated, unfrozen, or user send contexts. |
| 1108 | * User send contexts will be disabled when the process |
| 1109 | * calls into the driver to reset its context. |
| 1110 | */ |
| 1111 | if (!sc || !(sc->flags & SCF_FROZEN) || sc->type == SC_USER) |
| 1112 | continue; |
| 1113 | |
| 1114 | /* only need to disable, the context is already stopped */ |
| 1115 | sc_disable(sc); |
| 1116 | } |
| 1117 | } |
| 1118 | |
| 1119 | /* |
| 1120 | * Unfreeze PIO for kernel send contexts. The precondition for calling this |
| 1121 | * is that all PIO send contexts have been disabled and the SPC freeze has |
| 1122 | * been cleared. Now perform the last step and re-enable each kernel context. |
| 1123 | * User (PSM) processing will occur when PSM calls into the kernel to |
| 1124 | * acknowledge the freeze. |
| 1125 | */ |
| 1126 | void pio_kernel_unfreeze(struct hfi1_devdata *dd) |
| 1127 | { |
| 1128 | struct send_context *sc; |
| 1129 | int i; |
| 1130 | |
| 1131 | for (i = 0; i < dd->num_send_contexts; i++) { |
| 1132 | sc = dd->send_contexts[i].sc; |
| 1133 | if (!sc || !(sc->flags & SCF_FROZEN) || sc->type == SC_USER) |
| 1134 | continue; |
| 1135 | |
| 1136 | sc_enable(sc); /* will clear the sc frozen flag */ |
| 1137 | } |
| 1138 | } |
| 1139 | |
| 1140 | /* |
| 1141 | * Wait for the SendPioInitCtxt.PioInitInProgress bit to clear. |
| 1142 | * Returns: |
| 1143 | * -ETIMEDOUT - if we wait too long |
| 1144 | * -EIO - if there was an error |
| 1145 | */ |
| 1146 | static int pio_init_wait_progress(struct hfi1_devdata *dd) |
| 1147 | { |
| 1148 | u64 reg; |
| 1149 | int max, count = 0; |
| 1150 | |
| 1151 | /* max is the longest possible HW init time / delay */ |
| 1152 | max = (dd->icode == ICODE_FPGA_EMULATION) ? 120 : 5; |
| 1153 | while (1) { |
| 1154 | reg = read_csr(dd, SEND_PIO_INIT_CTXT); |
| 1155 | if (!(reg & SEND_PIO_INIT_CTXT_PIO_INIT_IN_PROGRESS_SMASK)) |
| 1156 | break; |
| 1157 | if (count >= max) |
| 1158 | return -ETIMEDOUT; |
| 1159 | udelay(5); |
| 1160 | count++; |
| 1161 | } |
| 1162 | |
| 1163 | return reg & SEND_PIO_INIT_CTXT_PIO_INIT_ERR_SMASK ? -EIO : 0; |
| 1164 | } |
| 1165 | |
| 1166 | /* |
| 1167 | * Reset all of the send contexts to their power-on state. Used |
| 1168 | * only during manual init - no lock against sc_enable needed. |
| 1169 | */ |
| 1170 | void pio_reset_all(struct hfi1_devdata *dd) |
| 1171 | { |
| 1172 | int ret; |
| 1173 | |
| 1174 | /* make sure the init engine is not busy */ |
| 1175 | ret = pio_init_wait_progress(dd); |
| 1176 | /* ignore any timeout */ |
| 1177 | if (ret == -EIO) { |
| 1178 | /* clear the error */ |
| 1179 | write_csr(dd, SEND_PIO_ERR_CLEAR, |
| 1180 | SEND_PIO_ERR_CLEAR_PIO_INIT_SM_IN_ERR_SMASK); |
| 1181 | } |
| 1182 | |
| 1183 | /* reset init all */ |
| 1184 | write_csr(dd, SEND_PIO_INIT_CTXT, |
| 1185 | SEND_PIO_INIT_CTXT_PIO_ALL_CTXT_INIT_SMASK); |
| 1186 | udelay(2); |
| 1187 | ret = pio_init_wait_progress(dd); |
| 1188 | if (ret < 0) { |
| 1189 | dd_dev_err(dd, |
| 1190 | "PIO send context init %s while initializing all PIO blocks\n", |
| 1191 | ret == -ETIMEDOUT ? "is stuck" : "had an error"); |
| 1192 | } |
| 1193 | } |
| 1194 | |
| 1195 | /* enable the context */ |
| 1196 | int sc_enable(struct send_context *sc) |
| 1197 | { |
| 1198 | u64 sc_ctrl, reg, pio; |
| 1199 | struct hfi1_devdata *dd; |
| 1200 | unsigned long flags; |
| 1201 | int ret = 0; |
| 1202 | |
| 1203 | if (!sc) |
| 1204 | return -EINVAL; |
| 1205 | dd = sc->dd; |
| 1206 | |
| 1207 | /* |
| 1208 | * Obtain the allocator lock to guard against any allocation |
| 1209 | * attempts (which should not happen prior to context being |
| 1210 | * enabled). On the release/disable side we don't need to |
| 1211 | * worry about locking since the releaser will not do anything |
| 1212 | * if the context accounting values have not changed. |
| 1213 | */ |
| 1214 | spin_lock_irqsave(&sc->alloc_lock, flags); |
| 1215 | sc_ctrl = read_kctxt_csr(dd, sc->hw_context, SC(CTRL)); |
| 1216 | if ((sc_ctrl & SC(CTRL_CTXT_ENABLE_SMASK))) |
| 1217 | goto unlock; /* already enabled */ |
| 1218 | |
| 1219 | /* IMPORTANT: only clear free and fill if transitioning 0 -> 1 */ |
| 1220 | |
| 1221 | *sc->hw_free = 0; |
| 1222 | sc->free = 0; |
| 1223 | sc->alloc_free = 0; |
| 1224 | sc->fill = 0; |
| 1225 | sc->sr_head = 0; |
| 1226 | sc->sr_tail = 0; |
| 1227 | sc->flags = 0; |
Mike Marciniszyn | a054374 | 2015-12-07 15:39:22 -0500 | [diff] [blame] | 1228 | /* the alloc lock insures no fast path allocation */ |
| 1229 | reset_buffers_allocated(sc); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 1230 | |
| 1231 | /* |
| 1232 | * Clear all per-context errors. Some of these will be set when |
| 1233 | * we are re-enabling after a context halt. Now that the context |
| 1234 | * is disabled, the halt will not clear until after the PIO init |
| 1235 | * engine runs below. |
| 1236 | */ |
| 1237 | reg = read_kctxt_csr(dd, sc->hw_context, SC(ERR_STATUS)); |
| 1238 | if (reg) |
| 1239 | write_kctxt_csr(dd, sc->hw_context, SC(ERR_CLEAR), |
| 1240 | reg); |
| 1241 | |
| 1242 | /* |
| 1243 | * The HW PIO initialization engine can handle only one init |
| 1244 | * request at a time. Serialize access to each device's engine. |
| 1245 | */ |
| 1246 | spin_lock(&dd->sc_init_lock); |
| 1247 | /* |
| 1248 | * Since access to this code block is serialized and |
| 1249 | * each access waits for the initialization to complete |
| 1250 | * before releasing the lock, the PIO initialization engine |
| 1251 | * should not be in use, so we don't have to wait for the |
| 1252 | * InProgress bit to go down. |
| 1253 | */ |
| 1254 | pio = ((sc->hw_context & SEND_PIO_INIT_CTXT_PIO_CTXT_NUM_MASK) << |
| 1255 | SEND_PIO_INIT_CTXT_PIO_CTXT_NUM_SHIFT) | |
| 1256 | SEND_PIO_INIT_CTXT_PIO_SINGLE_CTXT_INIT_SMASK; |
| 1257 | write_csr(dd, SEND_PIO_INIT_CTXT, pio); |
| 1258 | /* |
| 1259 | * Wait until the engine is done. Give the chip the required time |
| 1260 | * so, hopefully, we read the register just once. |
| 1261 | */ |
| 1262 | udelay(2); |
| 1263 | ret = pio_init_wait_progress(dd); |
| 1264 | spin_unlock(&dd->sc_init_lock); |
| 1265 | if (ret) { |
| 1266 | dd_dev_err(dd, |
| 1267 | "sctxt%u(%u): Context not enabled due to init failure %d\n", |
| 1268 | sc->sw_index, sc->hw_context, ret); |
| 1269 | goto unlock; |
| 1270 | } |
| 1271 | |
| 1272 | /* |
| 1273 | * All is well. Enable the context. |
| 1274 | */ |
| 1275 | sc_ctrl |= SC(CTRL_CTXT_ENABLE_SMASK); |
| 1276 | write_kctxt_csr(dd, sc->hw_context, SC(CTRL), sc_ctrl); |
| 1277 | /* |
| 1278 | * Read SendCtxtCtrl to force the write out and prevent a timing |
| 1279 | * hazard where a PIO write may reach the context before the enable. |
| 1280 | */ |
| 1281 | read_kctxt_csr(dd, sc->hw_context, SC(CTRL)); |
| 1282 | sc->flags |= SCF_ENABLED; |
| 1283 | |
| 1284 | unlock: |
| 1285 | spin_unlock_irqrestore(&sc->alloc_lock, flags); |
| 1286 | |
| 1287 | return ret; |
| 1288 | } |
| 1289 | |
| 1290 | /* force a credit return on the context */ |
| 1291 | void sc_return_credits(struct send_context *sc) |
| 1292 | { |
| 1293 | if (!sc) |
| 1294 | return; |
| 1295 | |
| 1296 | /* a 0->1 transition schedules a credit return */ |
| 1297 | write_kctxt_csr(sc->dd, sc->hw_context, SC(CREDIT_FORCE), |
| 1298 | SC(CREDIT_FORCE_FORCE_RETURN_SMASK)); |
| 1299 | /* |
| 1300 | * Ensure that the write is flushed and the credit return is |
| 1301 | * scheduled. We care more about the 0 -> 1 transition. |
| 1302 | */ |
| 1303 | read_kctxt_csr(sc->dd, sc->hw_context, SC(CREDIT_FORCE)); |
| 1304 | /* set back to 0 for next time */ |
| 1305 | write_kctxt_csr(sc->dd, sc->hw_context, SC(CREDIT_FORCE), 0); |
| 1306 | } |
| 1307 | |
| 1308 | /* allow all in-flight packets to drain on the context */ |
| 1309 | void sc_flush(struct send_context *sc) |
| 1310 | { |
| 1311 | if (!sc) |
| 1312 | return; |
| 1313 | |
| 1314 | sc_wait_for_packet_egress(sc, 1); |
| 1315 | } |
| 1316 | |
| 1317 | /* drop all packets on the context, no waiting until they are sent */ |
| 1318 | void sc_drop(struct send_context *sc) |
| 1319 | { |
| 1320 | if (!sc) |
| 1321 | return; |
| 1322 | |
| 1323 | dd_dev_info(sc->dd, "%s: context %u(%u) - not implemented\n", |
| 1324 | __func__, sc->sw_index, sc->hw_context); |
| 1325 | } |
| 1326 | |
| 1327 | /* |
| 1328 | * Start the software reaction to a context halt or SPC freeze: |
| 1329 | * - mark the context as halted or frozen |
| 1330 | * - stop buffer allocations |
| 1331 | * |
| 1332 | * Called from the error interrupt. Other work is deferred until |
| 1333 | * out of the interrupt. |
| 1334 | */ |
| 1335 | void sc_stop(struct send_context *sc, int flag) |
| 1336 | { |
| 1337 | unsigned long flags; |
| 1338 | |
| 1339 | /* mark the context */ |
| 1340 | sc->flags |= flag; |
| 1341 | |
| 1342 | /* stop buffer allocations */ |
| 1343 | spin_lock_irqsave(&sc->alloc_lock, flags); |
| 1344 | sc->flags &= ~SCF_ENABLED; |
| 1345 | spin_unlock_irqrestore(&sc->alloc_lock, flags); |
| 1346 | wake_up(&sc->halt_wait); |
| 1347 | } |
| 1348 | |
| 1349 | #define BLOCK_DWORDS (PIO_BLOCK_SIZE/sizeof(u32)) |
| 1350 | #define dwords_to_blocks(x) DIV_ROUND_UP(x, BLOCK_DWORDS) |
| 1351 | |
| 1352 | /* |
| 1353 | * The send context buffer "allocator". |
| 1354 | * |
| 1355 | * @sc: the PIO send context we are allocating from |
| 1356 | * @len: length of whole packet - including PBC - in dwords |
| 1357 | * @cb: optional callback to call when the buffer is finished sending |
| 1358 | * @arg: argument for cb |
| 1359 | * |
| 1360 | * Return a pointer to a PIO buffer if successful, NULL if not enough room. |
| 1361 | */ |
| 1362 | struct pio_buf *sc_buffer_alloc(struct send_context *sc, u32 dw_len, |
| 1363 | pio_release_cb cb, void *arg) |
| 1364 | { |
| 1365 | struct pio_buf *pbuf = NULL; |
| 1366 | unsigned long flags; |
| 1367 | unsigned long avail; |
| 1368 | unsigned long blocks = dwords_to_blocks(dw_len); |
| 1369 | unsigned long start_fill; |
| 1370 | int trycount = 0; |
| 1371 | u32 head, next; |
| 1372 | |
| 1373 | spin_lock_irqsave(&sc->alloc_lock, flags); |
| 1374 | if (!(sc->flags & SCF_ENABLED)) { |
| 1375 | spin_unlock_irqrestore(&sc->alloc_lock, flags); |
| 1376 | goto done; |
| 1377 | } |
| 1378 | |
| 1379 | retry: |
| 1380 | avail = (unsigned long)sc->credits - (sc->fill - sc->alloc_free); |
| 1381 | if (blocks > avail) { |
| 1382 | /* not enough room */ |
| 1383 | if (unlikely(trycount)) { /* already tried to get more room */ |
| 1384 | spin_unlock_irqrestore(&sc->alloc_lock, flags); |
| 1385 | goto done; |
| 1386 | } |
| 1387 | /* copy from receiver cache line and recalculate */ |
| 1388 | sc->alloc_free = ACCESS_ONCE(sc->free); |
| 1389 | avail = |
| 1390 | (unsigned long)sc->credits - |
| 1391 | (sc->fill - sc->alloc_free); |
| 1392 | if (blocks > avail) { |
| 1393 | /* still no room, actively update */ |
| 1394 | spin_unlock_irqrestore(&sc->alloc_lock, flags); |
| 1395 | sc_release_update(sc); |
| 1396 | spin_lock_irqsave(&sc->alloc_lock, flags); |
| 1397 | sc->alloc_free = ACCESS_ONCE(sc->free); |
| 1398 | trycount++; |
| 1399 | goto retry; |
| 1400 | } |
| 1401 | } |
| 1402 | |
| 1403 | /* there is enough room */ |
| 1404 | |
Mike Marciniszyn | a054374 | 2015-12-07 15:39:22 -0500 | [diff] [blame] | 1405 | preempt_disable(); |
| 1406 | this_cpu_inc(*sc->buffers_allocated); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 1407 | |
| 1408 | /* read this once */ |
| 1409 | head = sc->sr_head; |
| 1410 | |
| 1411 | /* "allocate" the buffer */ |
| 1412 | start_fill = sc->fill; |
| 1413 | sc->fill += blocks; |
| 1414 | |
| 1415 | /* |
| 1416 | * Fill the parts that the releaser looks at before moving the head. |
| 1417 | * The only necessary piece is the sent_at field. The credits |
| 1418 | * we have just allocated cannot have been returned yet, so the |
| 1419 | * cb and arg will not be looked at for a "while". Put them |
| 1420 | * on this side of the memory barrier anyway. |
| 1421 | */ |
| 1422 | pbuf = &sc->sr[head].pbuf; |
| 1423 | pbuf->sent_at = sc->fill; |
| 1424 | pbuf->cb = cb; |
| 1425 | pbuf->arg = arg; |
| 1426 | pbuf->sc = sc; /* could be filled in at sc->sr init time */ |
| 1427 | /* make sure this is in memory before updating the head */ |
| 1428 | |
| 1429 | /* calculate next head index, do not store */ |
| 1430 | next = head + 1; |
| 1431 | if (next >= sc->sr_size) |
| 1432 | next = 0; |
| 1433 | /* update the head - must be last! - the releaser can look at fields |
| 1434 | in pbuf once we move the head */ |
| 1435 | smp_wmb(); |
| 1436 | sc->sr_head = next; |
| 1437 | spin_unlock_irqrestore(&sc->alloc_lock, flags); |
| 1438 | |
| 1439 | /* finish filling in the buffer outside the lock */ |
| 1440 | pbuf->start = sc->base_addr + ((start_fill % sc->credits) |
| 1441 | * PIO_BLOCK_SIZE); |
| 1442 | pbuf->size = sc->credits * PIO_BLOCK_SIZE; |
| 1443 | pbuf->end = sc->base_addr + pbuf->size; |
| 1444 | pbuf->block_count = blocks; |
| 1445 | pbuf->qw_written = 0; |
| 1446 | pbuf->carry_bytes = 0; |
| 1447 | pbuf->carry.val64 = 0; |
| 1448 | done: |
| 1449 | return pbuf; |
| 1450 | } |
| 1451 | |
| 1452 | /* |
| 1453 | * There are at least two entities that can turn on credit return |
| 1454 | * interrupts and they can overlap. Avoid problems by implementing |
| 1455 | * a count scheme that is enforced by a lock. The lock is needed because |
| 1456 | * the count and CSR write must be paired. |
| 1457 | */ |
| 1458 | |
| 1459 | /* |
| 1460 | * Start credit return interrupts. This is managed by a count. If already |
| 1461 | * on, just increment the count. |
| 1462 | */ |
| 1463 | void sc_add_credit_return_intr(struct send_context *sc) |
| 1464 | { |
| 1465 | unsigned long flags; |
| 1466 | |
| 1467 | /* lock must surround both the count change and the CSR update */ |
| 1468 | spin_lock_irqsave(&sc->credit_ctrl_lock, flags); |
| 1469 | if (sc->credit_intr_count == 0) { |
| 1470 | sc->credit_ctrl |= SC(CREDIT_CTRL_CREDIT_INTR_SMASK); |
| 1471 | write_kctxt_csr(sc->dd, sc->hw_context, |
| 1472 | SC(CREDIT_CTRL), sc->credit_ctrl); |
| 1473 | } |
| 1474 | sc->credit_intr_count++; |
| 1475 | spin_unlock_irqrestore(&sc->credit_ctrl_lock, flags); |
| 1476 | } |
| 1477 | |
| 1478 | /* |
| 1479 | * Stop credit return interrupts. This is managed by a count. Decrement the |
| 1480 | * count, if the last user, then turn the credit interrupts off. |
| 1481 | */ |
| 1482 | void sc_del_credit_return_intr(struct send_context *sc) |
| 1483 | { |
| 1484 | unsigned long flags; |
| 1485 | |
| 1486 | WARN_ON(sc->credit_intr_count == 0); |
| 1487 | |
| 1488 | /* lock must surround both the count change and the CSR update */ |
| 1489 | spin_lock_irqsave(&sc->credit_ctrl_lock, flags); |
| 1490 | sc->credit_intr_count--; |
| 1491 | if (sc->credit_intr_count == 0) { |
| 1492 | sc->credit_ctrl &= ~SC(CREDIT_CTRL_CREDIT_INTR_SMASK); |
| 1493 | write_kctxt_csr(sc->dd, sc->hw_context, |
| 1494 | SC(CREDIT_CTRL), sc->credit_ctrl); |
| 1495 | } |
| 1496 | spin_unlock_irqrestore(&sc->credit_ctrl_lock, flags); |
| 1497 | } |
| 1498 | |
| 1499 | /* |
| 1500 | * The caller must be careful when calling this. All needint calls |
| 1501 | * must be paired with !needint. |
| 1502 | */ |
| 1503 | void hfi1_sc_wantpiobuf_intr(struct send_context *sc, u32 needint) |
| 1504 | { |
| 1505 | if (needint) |
| 1506 | sc_add_credit_return_intr(sc); |
| 1507 | else |
| 1508 | sc_del_credit_return_intr(sc); |
| 1509 | trace_hfi1_wantpiointr(sc, needint, sc->credit_ctrl); |
| 1510 | if (needint) { |
| 1511 | mmiowb(); |
| 1512 | sc_return_credits(sc); |
| 1513 | } |
| 1514 | } |
| 1515 | |
| 1516 | /** |
| 1517 | * sc_piobufavail - callback when a PIO buffer is available |
| 1518 | * @sc: the send context |
| 1519 | * |
| 1520 | * This is called from the interrupt handler when a PIO buffer is |
| 1521 | * available after hfi1_verbs_send() returned an error that no buffers were |
| 1522 | * available. Disable the interrupt if there are no more QPs waiting. |
| 1523 | */ |
| 1524 | static void sc_piobufavail(struct send_context *sc) |
| 1525 | { |
| 1526 | struct hfi1_devdata *dd = sc->dd; |
| 1527 | struct hfi1_ibdev *dev = &dd->verbs_dev; |
| 1528 | struct list_head *list; |
| 1529 | struct hfi1_qp *qps[PIO_WAIT_BATCH_SIZE]; |
| 1530 | struct hfi1_qp *qp; |
| 1531 | unsigned long flags; |
| 1532 | unsigned i, n = 0; |
| 1533 | |
| 1534 | if (dd->send_contexts[sc->sw_index].type != SC_KERNEL) |
| 1535 | return; |
| 1536 | list = &sc->piowait; |
| 1537 | /* |
| 1538 | * Note: checking that the piowait list is empty and clearing |
| 1539 | * the buffer available interrupt needs to be atomic or we |
| 1540 | * could end up with QPs on the wait list with the interrupt |
| 1541 | * disabled. |
| 1542 | */ |
| 1543 | write_seqlock_irqsave(&dev->iowait_lock, flags); |
| 1544 | while (!list_empty(list)) { |
| 1545 | struct iowait *wait; |
| 1546 | |
| 1547 | if (n == ARRAY_SIZE(qps)) |
| 1548 | goto full; |
| 1549 | wait = list_first_entry(list, struct iowait, list); |
| 1550 | qp = container_of(wait, struct hfi1_qp, s_iowait); |
| 1551 | list_del_init(&qp->s_iowait.list); |
| 1552 | /* refcount held until actual wake up */ |
| 1553 | qps[n++] = qp; |
| 1554 | } |
| 1555 | /* |
| 1556 | * Counting: only call wantpiobuf_intr() if there were waiters and they |
| 1557 | * are now all gone. |
| 1558 | */ |
| 1559 | if (n) |
| 1560 | hfi1_sc_wantpiobuf_intr(sc, 0); |
| 1561 | full: |
| 1562 | write_sequnlock_irqrestore(&dev->iowait_lock, flags); |
| 1563 | |
| 1564 | for (i = 0; i < n; i++) |
| 1565 | hfi1_qp_wakeup(qps[i], HFI1_S_WAIT_PIO); |
| 1566 | } |
| 1567 | |
| 1568 | /* translate a send credit update to a bit code of reasons */ |
| 1569 | static inline int fill_code(u64 hw_free) |
| 1570 | { |
| 1571 | int code = 0; |
| 1572 | |
| 1573 | if (hw_free & CR_STATUS_SMASK) |
| 1574 | code |= PRC_STATUS_ERR; |
| 1575 | if (hw_free & CR_CREDIT_RETURN_DUE_TO_PBC_SMASK) |
| 1576 | code |= PRC_PBC; |
| 1577 | if (hw_free & CR_CREDIT_RETURN_DUE_TO_THRESHOLD_SMASK) |
| 1578 | code |= PRC_THRESHOLD; |
| 1579 | if (hw_free & CR_CREDIT_RETURN_DUE_TO_ERR_SMASK) |
| 1580 | code |= PRC_FILL_ERR; |
| 1581 | if (hw_free & CR_CREDIT_RETURN_DUE_TO_FORCE_SMASK) |
| 1582 | code |= PRC_SC_DISABLE; |
| 1583 | return code; |
| 1584 | } |
| 1585 | |
| 1586 | /* use the jiffies compare to get the wrap right */ |
| 1587 | #define sent_before(a, b) time_before(a, b) /* a < b */ |
| 1588 | |
| 1589 | /* |
| 1590 | * The send context buffer "releaser". |
| 1591 | */ |
| 1592 | void sc_release_update(struct send_context *sc) |
| 1593 | { |
| 1594 | struct pio_buf *pbuf; |
| 1595 | u64 hw_free; |
| 1596 | u32 head, tail; |
| 1597 | unsigned long old_free; |
Mike Marciniszyn | e607a22 | 2015-12-03 14:34:18 -0500 | [diff] [blame] | 1598 | unsigned long free; |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 1599 | unsigned long extra; |
| 1600 | unsigned long flags; |
| 1601 | int code; |
| 1602 | |
| 1603 | if (!sc) |
| 1604 | return; |
| 1605 | |
| 1606 | spin_lock_irqsave(&sc->release_lock, flags); |
| 1607 | /* update free */ |
| 1608 | hw_free = le64_to_cpu(*sc->hw_free); /* volatile read */ |
| 1609 | old_free = sc->free; |
| 1610 | extra = (((hw_free & CR_COUNTER_SMASK) >> CR_COUNTER_SHIFT) |
| 1611 | - (old_free & CR_COUNTER_MASK)) |
| 1612 | & CR_COUNTER_MASK; |
Mike Marciniszyn | e607a22 | 2015-12-03 14:34:18 -0500 | [diff] [blame] | 1613 | free = old_free + extra; |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 1614 | trace_hfi1_piofree(sc, extra); |
| 1615 | |
| 1616 | /* call sent buffer callbacks */ |
| 1617 | code = -1; /* code not yet set */ |
| 1618 | head = ACCESS_ONCE(sc->sr_head); /* snapshot the head */ |
| 1619 | tail = sc->sr_tail; |
| 1620 | while (head != tail) { |
| 1621 | pbuf = &sc->sr[tail].pbuf; |
| 1622 | |
Mike Marciniszyn | e607a22 | 2015-12-03 14:34:18 -0500 | [diff] [blame] | 1623 | if (sent_before(free, pbuf->sent_at)) { |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 1624 | /* not sent yet */ |
| 1625 | break; |
| 1626 | } |
| 1627 | if (pbuf->cb) { |
| 1628 | if (code < 0) /* fill in code on first user */ |
| 1629 | code = fill_code(hw_free); |
| 1630 | (*pbuf->cb)(pbuf->arg, code); |
| 1631 | } |
| 1632 | |
| 1633 | tail++; |
| 1634 | if (tail >= sc->sr_size) |
| 1635 | tail = 0; |
| 1636 | } |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 1637 | sc->sr_tail = tail; |
Mike Marciniszyn | e607a22 | 2015-12-03 14:34:18 -0500 | [diff] [blame] | 1638 | /* make sure tail is updated before free */ |
| 1639 | smp_wmb(); |
| 1640 | sc->free = free; |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 1641 | spin_unlock_irqrestore(&sc->release_lock, flags); |
| 1642 | sc_piobufavail(sc); |
| 1643 | } |
| 1644 | |
| 1645 | /* |
| 1646 | * Send context group releaser. Argument is the send context that caused |
| 1647 | * the interrupt. Called from the send context interrupt handler. |
| 1648 | * |
| 1649 | * Call release on all contexts in the group. |
| 1650 | * |
| 1651 | * This routine takes the sc_lock without an irqsave because it is only |
| 1652 | * called from an interrupt handler. Adjust if that changes. |
| 1653 | */ |
| 1654 | void sc_group_release_update(struct hfi1_devdata *dd, u32 hw_context) |
| 1655 | { |
| 1656 | struct send_context *sc; |
| 1657 | u32 sw_index; |
| 1658 | u32 gc, gc_end; |
| 1659 | |
| 1660 | spin_lock(&dd->sc_lock); |
| 1661 | sw_index = dd->hw_to_sw[hw_context]; |
| 1662 | if (unlikely(sw_index >= dd->num_send_contexts)) { |
| 1663 | dd_dev_err(dd, "%s: invalid hw (%u) to sw (%u) mapping\n", |
| 1664 | __func__, hw_context, sw_index); |
| 1665 | goto done; |
| 1666 | } |
| 1667 | sc = dd->send_contexts[sw_index].sc; |
| 1668 | if (unlikely(!sc)) |
| 1669 | goto done; |
| 1670 | |
| 1671 | gc = group_context(hw_context, sc->group); |
| 1672 | gc_end = gc + group_size(sc->group); |
| 1673 | for (; gc < gc_end; gc++) { |
| 1674 | sw_index = dd->hw_to_sw[gc]; |
| 1675 | if (unlikely(sw_index >= dd->num_send_contexts)) { |
| 1676 | dd_dev_err(dd, |
| 1677 | "%s: invalid hw (%u) to sw (%u) mapping\n", |
| 1678 | __func__, hw_context, sw_index); |
| 1679 | continue; |
| 1680 | } |
| 1681 | sc_release_update(dd->send_contexts[sw_index].sc); |
| 1682 | } |
| 1683 | done: |
| 1684 | spin_unlock(&dd->sc_lock); |
| 1685 | } |
| 1686 | |
| 1687 | int init_pervl_scs(struct hfi1_devdata *dd) |
| 1688 | { |
| 1689 | int i; |
| 1690 | u64 mask, all_vl_mask = (u64) 0x80ff; /* VLs 0-7, 15 */ |
| 1691 | u32 ctxt; |
| 1692 | |
| 1693 | dd->vld[15].sc = sc_alloc(dd, SC_KERNEL, |
| 1694 | dd->rcd[0]->rcvhdrqentsize, dd->node); |
| 1695 | if (!dd->vld[15].sc) |
| 1696 | goto nomem; |
| 1697 | hfi1_init_ctxt(dd->vld[15].sc); |
| 1698 | dd->vld[15].mtu = enum_to_mtu(OPA_MTU_2048); |
| 1699 | for (i = 0; i < num_vls; i++) { |
| 1700 | /* |
| 1701 | * Since this function does not deal with a specific |
| 1702 | * receive context but we need the RcvHdrQ entry size, |
| 1703 | * use the size from rcd[0]. It is guaranteed to be |
| 1704 | * valid at this point and will remain the same for all |
| 1705 | * receive contexts. |
| 1706 | */ |
| 1707 | dd->vld[i].sc = sc_alloc(dd, SC_KERNEL, |
| 1708 | dd->rcd[0]->rcvhdrqentsize, dd->node); |
| 1709 | if (!dd->vld[i].sc) |
| 1710 | goto nomem; |
| 1711 | |
| 1712 | hfi1_init_ctxt(dd->vld[i].sc); |
| 1713 | |
| 1714 | /* non VL15 start with the max MTU */ |
| 1715 | dd->vld[i].mtu = hfi1_max_mtu; |
| 1716 | } |
| 1717 | sc_enable(dd->vld[15].sc); |
| 1718 | ctxt = dd->vld[15].sc->hw_context; |
| 1719 | mask = all_vl_mask & ~(1LL << 15); |
| 1720 | write_kctxt_csr(dd, ctxt, SC(CHECK_VL), mask); |
| 1721 | dd_dev_info(dd, |
| 1722 | "Using send context %u(%u) for VL15\n", |
| 1723 | dd->vld[15].sc->sw_index, ctxt); |
| 1724 | for (i = 0; i < num_vls; i++) { |
| 1725 | sc_enable(dd->vld[i].sc); |
| 1726 | ctxt = dd->vld[i].sc->hw_context; |
| 1727 | mask = all_vl_mask & ~(1LL << i); |
| 1728 | write_kctxt_csr(dd, ctxt, SC(CHECK_VL), mask); |
| 1729 | } |
| 1730 | return 0; |
| 1731 | nomem: |
| 1732 | sc_free(dd->vld[15].sc); |
| 1733 | for (i = 0; i < num_vls; i++) |
| 1734 | sc_free(dd->vld[i].sc); |
| 1735 | return -ENOMEM; |
| 1736 | } |
| 1737 | |
| 1738 | int init_credit_return(struct hfi1_devdata *dd) |
| 1739 | { |
| 1740 | int ret; |
| 1741 | int num_numa; |
| 1742 | int i; |
| 1743 | |
| 1744 | num_numa = num_online_nodes(); |
| 1745 | /* enforce the expectation that the numas are compact */ |
| 1746 | for (i = 0; i < num_numa; i++) { |
| 1747 | if (!node_online(i)) { |
| 1748 | dd_dev_err(dd, "NUMA nodes are not compact\n"); |
| 1749 | ret = -EINVAL; |
| 1750 | goto done; |
| 1751 | } |
| 1752 | } |
| 1753 | |
| 1754 | dd->cr_base = kcalloc( |
| 1755 | num_numa, |
| 1756 | sizeof(struct credit_return_base), |
| 1757 | GFP_KERNEL); |
| 1758 | if (!dd->cr_base) { |
| 1759 | dd_dev_err(dd, "Unable to allocate credit return base\n"); |
| 1760 | ret = -ENOMEM; |
| 1761 | goto done; |
| 1762 | } |
| 1763 | for (i = 0; i < num_numa; i++) { |
| 1764 | int bytes = TXE_NUM_CONTEXTS * sizeof(struct credit_return); |
| 1765 | |
| 1766 | set_dev_node(&dd->pcidev->dev, i); |
| 1767 | dd->cr_base[i].va = dma_zalloc_coherent( |
| 1768 | &dd->pcidev->dev, |
| 1769 | bytes, |
| 1770 | &dd->cr_base[i].pa, |
| 1771 | GFP_KERNEL); |
| 1772 | if (dd->cr_base[i].va == NULL) { |
| 1773 | set_dev_node(&dd->pcidev->dev, dd->node); |
| 1774 | dd_dev_err(dd, |
| 1775 | "Unable to allocate credit return DMA range for NUMA %d\n", |
| 1776 | i); |
| 1777 | ret = -ENOMEM; |
| 1778 | goto done; |
| 1779 | } |
| 1780 | } |
| 1781 | set_dev_node(&dd->pcidev->dev, dd->node); |
| 1782 | |
| 1783 | ret = 0; |
| 1784 | done: |
| 1785 | return ret; |
| 1786 | } |
| 1787 | |
| 1788 | void free_credit_return(struct hfi1_devdata *dd) |
| 1789 | { |
| 1790 | int num_numa; |
| 1791 | int i; |
| 1792 | |
| 1793 | if (!dd->cr_base) |
| 1794 | return; |
| 1795 | |
| 1796 | num_numa = num_online_nodes(); |
| 1797 | for (i = 0; i < num_numa; i++) { |
| 1798 | if (dd->cr_base[i].va) { |
| 1799 | dma_free_coherent(&dd->pcidev->dev, |
| 1800 | TXE_NUM_CONTEXTS |
| 1801 | * sizeof(struct credit_return), |
| 1802 | dd->cr_base[i].va, |
| 1803 | dd->cr_base[i].pa); |
| 1804 | } |
| 1805 | } |
| 1806 | kfree(dd->cr_base); |
| 1807 | dd->cr_base = NULL; |
| 1808 | } |