Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 1 | /* |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 2 | * This program is free software; you can redistribute it and/or modify |
| 3 | * it under the terms of the GNU General Public License as published by |
| 4 | * the Free Software Foundation; either version 2 of the License, or |
| 5 | * (at your option) any later version. |
| 6 | * |
| 7 | * This program is distributed in the hope that it will be useful, |
| 8 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 9 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 10 | * GNU General Public License for more details. |
| 11 | * |
| 12 | * You should have received a copy of the GNU General Public License |
| 13 | * along with this program; if not, write to the Free Software |
| 14 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
| 15 | * |
| 16 | * Copyright (c) 2005 Linas Vepstas <linas@linas.org> |
| 17 | */ |
| 18 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 19 | #include <linux/delay.h> |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 20 | #include <linux/list.h> |
Linas Vepstas | 8c33fd11 | 2006-03-29 15:29:18 -0600 | [diff] [blame] | 21 | #include <linux/mutex.h> |
Paul Gortmaker | 62fe91b | 2011-05-27 14:25:11 -0400 | [diff] [blame] | 22 | #include <linux/sched.h> |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 23 | #include <linux/pci.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 24 | #include <linux/slab.h> |
Linas Vepstas | 8c33fd11 | 2006-03-29 15:29:18 -0600 | [diff] [blame] | 25 | #include <linux/workqueue.h> |
Al Viro | ecf89e5 | 2012-10-02 15:32:10 -0400 | [diff] [blame^] | 26 | #include <linux/kthread.h> |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 27 | #include <asm/eeh_event.h> |
Linas Vepstas | 77bd741 | 2005-11-03 18:52:49 -0600 | [diff] [blame] | 28 | #include <asm/ppc-pci.h> |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 29 | |
| 30 | /** Overview: |
| 31 | * EEH error states may be detected within exception handlers; |
| 32 | * however, the recovery processing needs to occur asynchronously |
| 33 | * in a normal kernel context and not an interrupt context. |
| 34 | * This pair of routines creates an event and queues it onto a |
| 35 | * work-queue, where a worker thread can drive recovery. |
| 36 | */ |
| 37 | |
| 38 | /* EEH event workqueue setup. */ |
Ingo Molnar | 34af946 | 2006-06-27 02:53:55 -0700 | [diff] [blame] | 39 | static DEFINE_SPINLOCK(eeh_eventlist_lock); |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 40 | LIST_HEAD(eeh_eventlist); |
David Howells | c402895 | 2006-11-22 14:57:56 +0000 | [diff] [blame] | 41 | static void eeh_thread_launcher(struct work_struct *); |
| 42 | DECLARE_WORK(eeh_event_wq, eeh_thread_launcher); |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 43 | |
Linas Vepstas | 8c33fd11 | 2006-03-29 15:29:18 -0600 | [diff] [blame] | 44 | /* Serialize reset sequences for a given pci device */ |
| 45 | DEFINE_MUTEX(eeh_event_mutex); |
| 46 | |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 47 | /** |
Gavin Shan | 29f8bf1 | 2012-02-27 20:04:02 +0000 | [diff] [blame] | 48 | * eeh_event_handler - Dispatch EEH events. |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 49 | * @dummy - unused |
Linas Vepstas | 8c33fd11 | 2006-03-29 15:29:18 -0600 | [diff] [blame] | 50 | * |
| 51 | * The detection of a frozen slot can occur inside an interrupt, |
| 52 | * where it can be hard to do anything about it. The goal of this |
| 53 | * routine is to pull these detection events out of the context |
| 54 | * of the interrupt handler, and re-dispatch them for processing |
| 55 | * at a later time in a normal context. |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 56 | */ |
| 57 | static int eeh_event_handler(void * dummy) |
| 58 | { |
| 59 | unsigned long flags; |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 60 | struct eeh_event *event; |
| 61 | struct eeh_dev *edev; |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 62 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 63 | spin_lock_irqsave(&eeh_eventlist_lock, flags); |
| 64 | event = NULL; |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 65 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 66 | /* Unqueue the event, get ready to process. */ |
| 67 | if (!list_empty(&eeh_eventlist)) { |
| 68 | event = list_entry(eeh_eventlist.next, struct eeh_event, list); |
| 69 | list_del(&event->list); |
| 70 | } |
| 71 | spin_unlock_irqrestore(&eeh_eventlist_lock, flags); |
Linas Vepstas | 77bd741 | 2005-11-03 18:52:49 -0600 | [diff] [blame] | 72 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 73 | if (event == NULL) |
| 74 | return 0; |
Linas Vepstas | 8c33fd11 | 2006-03-29 15:29:18 -0600 | [diff] [blame] | 75 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 76 | /* Serialize processing of EEH events */ |
| 77 | mutex_lock(&eeh_event_mutex); |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 78 | edev = event->edev; |
| 79 | eeh_mark_slot(eeh_dev_to_of_node(edev), EEH_MODE_RECOVERING); |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 80 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 81 | printk(KERN_INFO "EEH: Detected PCI bus error on device %s\n", |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 82 | eeh_pci_name(edev->pdev)); |
Linas Vepstas | 8c33fd11 | 2006-03-29 15:29:18 -0600 | [diff] [blame] | 83 | |
Andrew Morton | 9b218f6 | 2012-03-28 12:20:58 +0000 | [diff] [blame] | 84 | set_current_state(TASK_INTERRUPTIBLE); /* Don't add to load average */ |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 85 | edev = handle_eeh_events(event); |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 86 | |
Kleber Sacilotto de Souza | 10db8d2 | 2012-07-12 17:14:36 +0000 | [diff] [blame] | 87 | if (edev) { |
| 88 | eeh_clear_slot(eeh_dev_to_of_node(edev), EEH_MODE_RECOVERING); |
| 89 | pci_dev_put(edev->pdev); |
| 90 | } |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 91 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 92 | kfree(event); |
| 93 | mutex_unlock(&eeh_event_mutex); |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 94 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 95 | /* If there are no new errors after an hour, clear the counter. */ |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 96 | if (edev && edev->freeze_count>0) { |
Gavin Shan | 29f8bf1 | 2012-02-27 20:04:02 +0000 | [diff] [blame] | 97 | msleep_interruptible(3600*1000); |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 98 | if (edev->freeze_count>0) |
| 99 | edev->freeze_count--; |
| 100 | |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 101 | } |
| 102 | |
| 103 | return 0; |
| 104 | } |
| 105 | |
| 106 | /** |
Gavin Shan | 29f8bf1 | 2012-02-27 20:04:02 +0000 | [diff] [blame] | 107 | * eeh_thread_launcher - Start kernel thread to handle EEH events |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 108 | * @dummy - unused |
Gavin Shan | 29f8bf1 | 2012-02-27 20:04:02 +0000 | [diff] [blame] | 109 | * |
| 110 | * This routine is called to start the kernel thread for processing |
| 111 | * EEH event. |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 112 | */ |
David Howells | c402895 | 2006-11-22 14:57:56 +0000 | [diff] [blame] | 113 | static void eeh_thread_launcher(struct work_struct *dummy) |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 114 | { |
Al Viro | ecf89e5 | 2012-10-02 15:32:10 -0400 | [diff] [blame^] | 115 | if (IS_ERR(kthread_run(eeh_event_handler, NULL, "eehd"))) |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 116 | printk(KERN_ERR "Failed to start EEH daemon\n"); |
| 117 | } |
| 118 | |
| 119 | /** |
Gavin Shan | 29f8bf1 | 2012-02-27 20:04:02 +0000 | [diff] [blame] | 120 | * eeh_send_failure_event - Generate a PCI error event |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 121 | * @edev: EEH device |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 122 | * |
| 123 | * This routine can be called within an interrupt context; |
| 124 | * the actual event will be delivered in a normal context |
| 125 | * (from a workqueue). |
| 126 | */ |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 127 | int eeh_send_failure_event(struct eeh_dev *edev) |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 128 | { |
| 129 | unsigned long flags; |
| 130 | struct eeh_event *event; |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 131 | struct device_node *dn = eeh_dev_to_of_node(edev); |
Jeremy Kerr | 954a46e | 2006-07-12 15:39:43 +1000 | [diff] [blame] | 132 | const char *location; |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 133 | |
Linas Vepstas | 054d8ff | 2006-04-27 02:31:20 -0700 | [diff] [blame] | 134 | if (!mem_init_done) { |
| 135 | printk(KERN_ERR "EEH: event during early boot not handled\n"); |
Stephen Rothwell | e2eb639 | 2007-04-03 22:26:41 +1000 | [diff] [blame] | 136 | location = of_get_property(dn, "ibm,loc-code", NULL); |
Linas Vepstas | 054d8ff | 2006-04-27 02:31:20 -0700 | [diff] [blame] | 137 | printk(KERN_ERR "EEH: device node = %s\n", dn->full_name); |
| 138 | printk(KERN_ERR "EEH: PCI location = %s\n", location); |
| 139 | return 1; |
| 140 | } |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 141 | event = kmalloc(sizeof(*event), GFP_ATOMIC); |
| 142 | if (event == NULL) { |
Gavin Shan | 29f8bf1 | 2012-02-27 20:04:02 +0000 | [diff] [blame] | 143 | printk(KERN_ERR "EEH: out of memory, event not handled\n"); |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 144 | return 1; |
| 145 | } |
| 146 | |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 147 | if (edev->pdev) |
| 148 | pci_dev_get(edev->pdev); |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 149 | |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 150 | event->edev = edev; |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 151 | |
| 152 | /* We may or may not be called in an interrupt context */ |
| 153 | spin_lock_irqsave(&eeh_eventlist_lock, flags); |
| 154 | list_add(&event->list, &eeh_eventlist); |
| 155 | spin_unlock_irqrestore(&eeh_eventlist_lock, flags); |
| 156 | |
| 157 | schedule_work(&eeh_event_wq); |
| 158 | |
| 159 | return 0; |
| 160 | } |