Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 1 | /* |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 2 | * This program is free software; you can redistribute it and/or modify |
| 3 | * it under the terms of the GNU General Public License as published by |
| 4 | * the Free Software Foundation; either version 2 of the License, or |
| 5 | * (at your option) any later version. |
| 6 | * |
| 7 | * This program is distributed in the hope that it will be useful, |
| 8 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 9 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 10 | * GNU General Public License for more details. |
| 11 | * |
| 12 | * You should have received a copy of the GNU General Public License |
| 13 | * along with this program; if not, write to the Free Software |
| 14 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
| 15 | * |
| 16 | * Copyright (c) 2005 Linas Vepstas <linas@linas.org> |
| 17 | */ |
| 18 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 19 | #include <linux/delay.h> |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 20 | #include <linux/list.h> |
Linas Vepstas | 8c33fd11 | 2006-03-29 15:29:18 -0600 | [diff] [blame] | 21 | #include <linux/mutex.h> |
Paul Gortmaker | 62fe91b | 2011-05-27 14:25:11 -0400 | [diff] [blame] | 22 | #include <linux/sched.h> |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 23 | #include <linux/pci.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 24 | #include <linux/slab.h> |
Linas Vepstas | 8c33fd11 | 2006-03-29 15:29:18 -0600 | [diff] [blame] | 25 | #include <linux/workqueue.h> |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 26 | #include <asm/eeh_event.h> |
Linas Vepstas | 77bd741 | 2005-11-03 18:52:49 -0600 | [diff] [blame] | 27 | #include <asm/ppc-pci.h> |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 28 | |
| 29 | /** Overview: |
| 30 | * EEH error states may be detected within exception handlers; |
| 31 | * however, the recovery processing needs to occur asynchronously |
| 32 | * in a normal kernel context and not an interrupt context. |
| 33 | * This pair of routines creates an event and queues it onto a |
| 34 | * work-queue, where a worker thread can drive recovery. |
| 35 | */ |
| 36 | |
| 37 | /* EEH event workqueue setup. */ |
Ingo Molnar | 34af946 | 2006-06-27 02:53:55 -0700 | [diff] [blame] | 38 | static DEFINE_SPINLOCK(eeh_eventlist_lock); |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 39 | LIST_HEAD(eeh_eventlist); |
David Howells | c402895 | 2006-11-22 14:57:56 +0000 | [diff] [blame] | 40 | static void eeh_thread_launcher(struct work_struct *); |
| 41 | DECLARE_WORK(eeh_event_wq, eeh_thread_launcher); |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 42 | |
Linas Vepstas | 8c33fd11 | 2006-03-29 15:29:18 -0600 | [diff] [blame] | 43 | /* Serialize reset sequences for a given pci device */ |
| 44 | DEFINE_MUTEX(eeh_event_mutex); |
| 45 | |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 46 | /** |
Gavin Shan | 29f8bf1 | 2012-02-27 20:04:02 +0000 | [diff] [blame] | 47 | * eeh_event_handler - Dispatch EEH events. |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 48 | * @dummy - unused |
Linas Vepstas | 8c33fd11 | 2006-03-29 15:29:18 -0600 | [diff] [blame] | 49 | * |
| 50 | * The detection of a frozen slot can occur inside an interrupt, |
| 51 | * where it can be hard to do anything about it. The goal of this |
| 52 | * routine is to pull these detection events out of the context |
| 53 | * of the interrupt handler, and re-dispatch them for processing |
| 54 | * at a later time in a normal context. |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 55 | */ |
| 56 | static int eeh_event_handler(void * dummy) |
| 57 | { |
| 58 | unsigned long flags; |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 59 | struct eeh_event *event; |
| 60 | struct eeh_dev *edev; |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 61 | |
Oleg Nesterov | 37ef9bd | 2012-03-28 12:20:57 +0000 | [diff] [blame] | 62 | set_task_comm(current, "eehd"); |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 63 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 64 | spin_lock_irqsave(&eeh_eventlist_lock, flags); |
| 65 | event = NULL; |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 66 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 67 | /* Unqueue the event, get ready to process. */ |
| 68 | if (!list_empty(&eeh_eventlist)) { |
| 69 | event = list_entry(eeh_eventlist.next, struct eeh_event, list); |
| 70 | list_del(&event->list); |
| 71 | } |
| 72 | spin_unlock_irqrestore(&eeh_eventlist_lock, flags); |
Linas Vepstas | 77bd741 | 2005-11-03 18:52:49 -0600 | [diff] [blame] | 73 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 74 | if (event == NULL) |
| 75 | return 0; |
Linas Vepstas | 8c33fd11 | 2006-03-29 15:29:18 -0600 | [diff] [blame] | 76 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 77 | /* Serialize processing of EEH events */ |
| 78 | mutex_lock(&eeh_event_mutex); |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 79 | edev = event->edev; |
| 80 | eeh_mark_slot(eeh_dev_to_of_node(edev), EEH_MODE_RECOVERING); |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 81 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 82 | printk(KERN_INFO "EEH: Detected PCI bus error on device %s\n", |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 83 | eeh_pci_name(edev->pdev)); |
Linas Vepstas | 8c33fd11 | 2006-03-29 15:29:18 -0600 | [diff] [blame] | 84 | |
Andrew Morton | 9b218f6 | 2012-03-28 12:20:58 +0000 | [diff] [blame] | 85 | set_current_state(TASK_INTERRUPTIBLE); /* Don't add to load average */ |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 86 | edev = handle_eeh_events(event); |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 87 | |
Kleber Sacilotto de Souza | 10db8d2 | 2012-07-12 17:14:36 +0000 | [diff] [blame] | 88 | if (edev) { |
| 89 | eeh_clear_slot(eeh_dev_to_of_node(edev), EEH_MODE_RECOVERING); |
| 90 | pci_dev_put(edev->pdev); |
| 91 | } |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 92 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 93 | kfree(event); |
| 94 | mutex_unlock(&eeh_event_mutex); |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 95 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 96 | /* If there are no new errors after an hour, clear the counter. */ |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 97 | if (edev && edev->freeze_count>0) { |
Gavin Shan | 29f8bf1 | 2012-02-27 20:04:02 +0000 | [diff] [blame] | 98 | msleep_interruptible(3600*1000); |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 99 | if (edev->freeze_count>0) |
| 100 | edev->freeze_count--; |
| 101 | |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 102 | } |
| 103 | |
| 104 | return 0; |
| 105 | } |
| 106 | |
| 107 | /** |
Gavin Shan | 29f8bf1 | 2012-02-27 20:04:02 +0000 | [diff] [blame] | 108 | * eeh_thread_launcher - Start kernel thread to handle EEH events |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 109 | * @dummy - unused |
Gavin Shan | 29f8bf1 | 2012-02-27 20:04:02 +0000 | [diff] [blame] | 110 | * |
| 111 | * This routine is called to start the kernel thread for processing |
| 112 | * EEH event. |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 113 | */ |
David Howells | c402895 | 2006-11-22 14:57:56 +0000 | [diff] [blame] | 114 | static void eeh_thread_launcher(struct work_struct *dummy) |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 115 | { |
| 116 | if (kernel_thread(eeh_event_handler, NULL, CLONE_KERNEL) < 0) |
| 117 | printk(KERN_ERR "Failed to start EEH daemon\n"); |
| 118 | } |
| 119 | |
| 120 | /** |
Gavin Shan | 29f8bf1 | 2012-02-27 20:04:02 +0000 | [diff] [blame] | 121 | * eeh_send_failure_event - Generate a PCI error event |
Gavin Shan | c533b46 | 2012-09-07 22:44:11 +0000 | [diff] [blame^] | 122 | * @pe: EEH PE |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 123 | * |
| 124 | * This routine can be called within an interrupt context; |
| 125 | * the actual event will be delivered in a normal context |
| 126 | * (from a workqueue). |
| 127 | */ |
Gavin Shan | c533b46 | 2012-09-07 22:44:11 +0000 | [diff] [blame^] | 128 | int eeh_send_failure_event(struct eeh_pe *pe) |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 129 | { |
| 130 | unsigned long flags; |
| 131 | struct eeh_event *event; |
| 132 | |
Gavin Shan | 7e4bbaf | 2012-09-07 22:44:03 +0000 | [diff] [blame] | 133 | event = kzalloc(sizeof(*event), GFP_ATOMIC); |
Gavin Shan | c533b46 | 2012-09-07 22:44:11 +0000 | [diff] [blame^] | 134 | if (!event) { |
| 135 | pr_err("EEH: out of memory, event not handled\n"); |
| 136 | return -ENOMEM; |
| 137 | } |
| 138 | event->pe = pe; |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 139 | |
| 140 | /* We may or may not be called in an interrupt context */ |
| 141 | spin_lock_irqsave(&eeh_eventlist_lock, flags); |
| 142 | list_add(&event->list, &eeh_eventlist); |
| 143 | spin_unlock_irqrestore(&eeh_eventlist_lock, flags); |
| 144 | |
| 145 | schedule_work(&eeh_event_wq); |
| 146 | |
| 147 | return 0; |
| 148 | } |