Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1 | /* |
Saeed Mahameed | 302bdf6 | 2015-04-02 17:07:29 +0300 | [diff] [blame] | 2 | * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 3 | * |
| 4 | * This software is available to you under a choice of one of two |
| 5 | * licenses. You may choose to be licensed under the terms of the GNU |
| 6 | * General Public License (GPL) Version 2, available from the file |
| 7 | * COPYING in the main directory of this source tree, or the |
| 8 | * OpenIB.org BSD license below: |
| 9 | * |
| 10 | * Redistribution and use in source and binary forms, with or |
| 11 | * without modification, are permitted provided that the following |
| 12 | * conditions are met: |
| 13 | * |
| 14 | * - Redistributions of source code must retain the above |
| 15 | * copyright notice, this list of conditions and the following |
| 16 | * disclaimer. |
| 17 | * |
| 18 | * - Redistributions in binary form must reproduce the above |
| 19 | * copyright notice, this list of conditions and the following |
| 20 | * disclaimer in the documentation and/or other materials |
| 21 | * provided with the distribution. |
| 22 | * |
| 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 30 | * SOFTWARE. |
| 31 | */ |
| 32 | |
| 33 | #include <linux/kernel.h> |
| 34 | #include <linux/module.h> |
| 35 | #include <linux/random.h> |
| 36 | #include <linux/vmalloc.h> |
| 37 | #include <linux/mlx5/driver.h> |
| 38 | #include <linux/mlx5/cmd.h> |
| 39 | #include "mlx5_core.h" |
| 40 | |
| 41 | enum { |
| 42 | MLX5_HEALTH_POLL_INTERVAL = 2 * HZ, |
| 43 | MAX_MISSES = 3, |
| 44 | }; |
| 45 | |
| 46 | enum { |
| 47 | MLX5_HEALTH_SYNDR_FW_ERR = 0x1, |
| 48 | MLX5_HEALTH_SYNDR_IRISC_ERR = 0x7, |
Eli Cohen | 171bb2c | 2015-09-25 10:49:16 +0300 | [diff] [blame] | 49 | MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR = 0x8, |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 50 | MLX5_HEALTH_SYNDR_CRC_ERR = 0x9, |
| 51 | MLX5_HEALTH_SYNDR_FETCH_PCI_ERR = 0xa, |
| 52 | MLX5_HEALTH_SYNDR_HW_FTL_ERR = 0xb, |
| 53 | MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR = 0xc, |
| 54 | MLX5_HEALTH_SYNDR_EQ_ERR = 0xd, |
Eli Cohen | 171bb2c | 2015-09-25 10:49:16 +0300 | [diff] [blame] | 55 | MLX5_HEALTH_SYNDR_EQ_INV = 0xe, |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 56 | MLX5_HEALTH_SYNDR_FFSER_ERR = 0xf, |
Eli Cohen | 171bb2c | 2015-09-25 10:49:16 +0300 | [diff] [blame] | 57 | MLX5_HEALTH_SYNDR_HIGH_TEMP = 0x10 |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 58 | }; |
| 59 | |
| 60 | static DEFINE_SPINLOCK(health_lock); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 61 | static LIST_HEAD(health_list); |
| 62 | static struct work_struct health_work; |
| 63 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 64 | static void health_care(struct work_struct *work) |
| 65 | { |
| 66 | struct mlx5_core_health *health, *n; |
| 67 | struct mlx5_core_dev *dev; |
| 68 | struct mlx5_priv *priv; |
| 69 | LIST_HEAD(tlist); |
| 70 | |
| 71 | spin_lock_irq(&health_lock); |
| 72 | list_splice_init(&health_list, &tlist); |
| 73 | |
| 74 | spin_unlock_irq(&health_lock); |
| 75 | |
| 76 | list_for_each_entry_safe(health, n, &tlist, list) { |
| 77 | priv = container_of(health, struct mlx5_priv, health); |
| 78 | dev = container_of(priv, struct mlx5_core_dev, priv); |
| 79 | mlx5_core_warn(dev, "handling bad device here\n"); |
Eli Cohen | 7d46dab | 2013-08-05 16:05:32 +0300 | [diff] [blame] | 80 | /* nothing yet */ |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 81 | spin_lock_irq(&health_lock); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 82 | list_del_init(&health->list); |
| 83 | spin_unlock_irq(&health_lock); |
| 84 | } |
| 85 | } |
| 86 | |
| 87 | static const char *hsynd_str(u8 synd) |
| 88 | { |
| 89 | switch (synd) { |
| 90 | case MLX5_HEALTH_SYNDR_FW_ERR: |
| 91 | return "firmware internal error"; |
| 92 | case MLX5_HEALTH_SYNDR_IRISC_ERR: |
| 93 | return "irisc not responding"; |
Eli Cohen | 171bb2c | 2015-09-25 10:49:16 +0300 | [diff] [blame] | 94 | case MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR: |
| 95 | return "unrecoverable hardware error"; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 96 | case MLX5_HEALTH_SYNDR_CRC_ERR: |
| 97 | return "firmware CRC error"; |
| 98 | case MLX5_HEALTH_SYNDR_FETCH_PCI_ERR: |
| 99 | return "ICM fetch PCI error"; |
| 100 | case MLX5_HEALTH_SYNDR_HW_FTL_ERR: |
| 101 | return "HW fatal error\n"; |
| 102 | case MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR: |
| 103 | return "async EQ buffer overrun"; |
| 104 | case MLX5_HEALTH_SYNDR_EQ_ERR: |
| 105 | return "EQ error"; |
Eli Cohen | 171bb2c | 2015-09-25 10:49:16 +0300 | [diff] [blame] | 106 | case MLX5_HEALTH_SYNDR_EQ_INV: |
| 107 | return "Invalid EQ refrenced"; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 108 | case MLX5_HEALTH_SYNDR_FFSER_ERR: |
| 109 | return "FFSER error"; |
Eli Cohen | 171bb2c | 2015-09-25 10:49:16 +0300 | [diff] [blame] | 110 | case MLX5_HEALTH_SYNDR_HIGH_TEMP: |
| 111 | return "High temprature"; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 112 | default: |
| 113 | return "unrecognized error"; |
| 114 | } |
| 115 | } |
| 116 | |
Roland Dreier | 582c016 | 2013-07-08 10:52:28 -0700 | [diff] [blame] | 117 | static u16 read_be16(__be16 __iomem *p) |
| 118 | { |
| 119 | return swab16(readl((__force u16 __iomem *) p)); |
| 120 | } |
| 121 | |
| 122 | static u32 read_be32(__be32 __iomem *p) |
| 123 | { |
| 124 | return swab32(readl((__force u32 __iomem *) p)); |
| 125 | } |
| 126 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 127 | static void print_health_info(struct mlx5_core_dev *dev) |
| 128 | { |
| 129 | struct mlx5_core_health *health = &dev->priv.health; |
| 130 | struct health_buffer __iomem *h = health->health; |
| 131 | int i; |
| 132 | |
| 133 | for (i = 0; i < ARRAY_SIZE(h->assert_var); i++) |
Roland Dreier | 582c016 | 2013-07-08 10:52:28 -0700 | [diff] [blame] | 134 | pr_info("assert_var[%d] 0x%08x\n", i, read_be32(h->assert_var + i)); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 135 | |
Roland Dreier | 582c016 | 2013-07-08 10:52:28 -0700 | [diff] [blame] | 136 | pr_info("assert_exit_ptr 0x%08x\n", read_be32(&h->assert_exit_ptr)); |
| 137 | pr_info("assert_callra 0x%08x\n", read_be32(&h->assert_callra)); |
| 138 | pr_info("fw_ver 0x%08x\n", read_be32(&h->fw_ver)); |
| 139 | pr_info("hw_id 0x%08x\n", read_be32(&h->hw_id)); |
| 140 | pr_info("irisc_index %d\n", readb(&h->irisc_index)); |
| 141 | pr_info("synd 0x%x: %s\n", readb(&h->synd), hsynd_str(readb(&h->synd))); |
Eli Cohen | 78ccb25 | 2015-09-25 10:49:15 +0300 | [diff] [blame] | 142 | pr_info("ext_sync 0x%04x\n", read_be16(&h->ext_synd)); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 143 | } |
| 144 | |
| 145 | static void poll_health(unsigned long data) |
| 146 | { |
| 147 | struct mlx5_core_dev *dev = (struct mlx5_core_dev *)data; |
| 148 | struct mlx5_core_health *health = &dev->priv.health; |
| 149 | unsigned long next; |
| 150 | u32 count; |
| 151 | |
| 152 | count = ioread32be(health->health_counter); |
| 153 | if (count == health->prev) |
| 154 | ++health->miss_counter; |
| 155 | else |
| 156 | health->miss_counter = 0; |
| 157 | |
| 158 | health->prev = count; |
| 159 | if (health->miss_counter == MAX_MISSES) { |
| 160 | mlx5_core_err(dev, "device's health compromised\n"); |
| 161 | print_health_info(dev); |
| 162 | spin_lock_irq(&health_lock); |
| 163 | list_add_tail(&health->list, &health_list); |
| 164 | spin_unlock_irq(&health_lock); |
| 165 | |
| 166 | queue_work(mlx5_core_wq, &health_work); |
| 167 | } else { |
| 168 | get_random_bytes(&next, sizeof(next)); |
| 169 | next %= HZ; |
| 170 | next += jiffies + MLX5_HEALTH_POLL_INTERVAL; |
| 171 | mod_timer(&health->timer, next); |
| 172 | } |
| 173 | } |
| 174 | |
| 175 | void mlx5_start_health_poll(struct mlx5_core_dev *dev) |
| 176 | { |
| 177 | struct mlx5_core_health *health = &dev->priv.health; |
| 178 | |
| 179 | INIT_LIST_HEAD(&health->list); |
| 180 | init_timer(&health->timer); |
| 181 | health->health = &dev->iseg->health; |
| 182 | health->health_counter = &dev->iseg->health_counter; |
| 183 | |
| 184 | health->timer.data = (unsigned long)dev; |
| 185 | health->timer.function = poll_health; |
| 186 | health->timer.expires = round_jiffies(jiffies + MLX5_HEALTH_POLL_INTERVAL); |
| 187 | add_timer(&health->timer); |
| 188 | } |
| 189 | |
| 190 | void mlx5_stop_health_poll(struct mlx5_core_dev *dev) |
| 191 | { |
| 192 | struct mlx5_core_health *health = &dev->priv.health; |
| 193 | |
| 194 | del_timer_sync(&health->timer); |
| 195 | |
| 196 | spin_lock_irq(&health_lock); |
| 197 | if (!list_empty(&health->list)) |
| 198 | list_del_init(&health->list); |
| 199 | spin_unlock_irq(&health_lock); |
| 200 | } |
| 201 | |
| 202 | void mlx5_health_cleanup(void) |
| 203 | { |
| 204 | } |
| 205 | |
| 206 | void __init mlx5_health_init(void) |
| 207 | { |
| 208 | INIT_WORK(&health_work, health_care); |
| 209 | } |