Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 1 | /* |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 2 | * This program is free software; you can redistribute it and/or modify |
| 3 | * it under the terms of the GNU General Public License as published by |
| 4 | * the Free Software Foundation; either version 2 of the License, or |
| 5 | * (at your option) any later version. |
| 6 | * |
| 7 | * This program is distributed in the hope that it will be useful, |
| 8 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 9 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 10 | * GNU General Public License for more details. |
| 11 | * |
| 12 | * You should have received a copy of the GNU General Public License |
| 13 | * along with this program; if not, write to the Free Software |
| 14 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
| 15 | * |
| 16 | * Copyright (c) 2005 Linas Vepstas <linas@linas.org> |
| 17 | */ |
| 18 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 19 | #include <linux/delay.h> |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 20 | #include <linux/list.h> |
Linas Vepstas | 8c33fd11 | 2006-03-29 15:29:18 -0600 | [diff] [blame] | 21 | #include <linux/mutex.h> |
Paul Gortmaker | 62fe91b | 2011-05-27 14:25:11 -0400 | [diff] [blame] | 22 | #include <linux/sched.h> |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 23 | #include <linux/pci.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 24 | #include <linux/slab.h> |
Linas Vepstas | 8c33fd11 | 2006-03-29 15:29:18 -0600 | [diff] [blame] | 25 | #include <linux/workqueue.h> |
Al Viro | ecf89e5 | 2012-10-02 15:32:10 -0400 | [diff] [blame] | 26 | #include <linux/kthread.h> |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 27 | #include <asm/eeh_event.h> |
Linas Vepstas | 77bd741 | 2005-11-03 18:52:49 -0600 | [diff] [blame] | 28 | #include <asm/ppc-pci.h> |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 29 | |
| 30 | /** Overview: |
| 31 | * EEH error states may be detected within exception handlers; |
| 32 | * however, the recovery processing needs to occur asynchronously |
| 33 | * in a normal kernel context and not an interrupt context. |
| 34 | * This pair of routines creates an event and queues it onto a |
| 35 | * work-queue, where a worker thread can drive recovery. |
| 36 | */ |
| 37 | |
| 38 | /* EEH event workqueue setup. */ |
Ingo Molnar | 34af946 | 2006-06-27 02:53:55 -0700 | [diff] [blame] | 39 | static DEFINE_SPINLOCK(eeh_eventlist_lock); |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 40 | LIST_HEAD(eeh_eventlist); |
David Howells | c402895 | 2006-11-22 14:57:56 +0000 | [diff] [blame] | 41 | static void eeh_thread_launcher(struct work_struct *); |
| 42 | DECLARE_WORK(eeh_event_wq, eeh_thread_launcher); |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 43 | |
Linas Vepstas | 8c33fd11 | 2006-03-29 15:29:18 -0600 | [diff] [blame] | 44 | /* Serialize reset sequences for a given pci device */ |
| 45 | DEFINE_MUTEX(eeh_event_mutex); |
| 46 | |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 47 | /** |
Gavin Shan | 29f8bf1 | 2012-02-27 20:04:02 +0000 | [diff] [blame] | 48 | * eeh_event_handler - Dispatch EEH events. |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 49 | * @dummy - unused |
Linas Vepstas | 8c33fd11 | 2006-03-29 15:29:18 -0600 | [diff] [blame] | 50 | * |
| 51 | * The detection of a frozen slot can occur inside an interrupt, |
| 52 | * where it can be hard to do anything about it. The goal of this |
| 53 | * routine is to pull these detection events out of the context |
| 54 | * of the interrupt handler, and re-dispatch them for processing |
| 55 | * at a later time in a normal context. |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 56 | */ |
| 57 | static int eeh_event_handler(void * dummy) |
| 58 | { |
| 59 | unsigned long flags; |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 60 | struct eeh_event *event; |
Gavin Shan | 120dc49 | 2012-09-07 22:44:18 +0000 | [diff] [blame] | 61 | struct eeh_pe *pe; |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 62 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 63 | spin_lock_irqsave(&eeh_eventlist_lock, flags); |
| 64 | event = NULL; |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 65 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 66 | /* Unqueue the event, get ready to process. */ |
| 67 | if (!list_empty(&eeh_eventlist)) { |
| 68 | event = list_entry(eeh_eventlist.next, struct eeh_event, list); |
| 69 | list_del(&event->list); |
| 70 | } |
| 71 | spin_unlock_irqrestore(&eeh_eventlist_lock, flags); |
Linas Vepstas | 77bd741 | 2005-11-03 18:52:49 -0600 | [diff] [blame] | 72 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 73 | if (event == NULL) |
| 74 | return 0; |
Linas Vepstas | 8c33fd11 | 2006-03-29 15:29:18 -0600 | [diff] [blame] | 75 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 76 | /* Serialize processing of EEH events */ |
| 77 | mutex_lock(&eeh_event_mutex); |
Gavin Shan | 120dc49 | 2012-09-07 22:44:18 +0000 | [diff] [blame] | 78 | pe = event->pe; |
| 79 | eeh_pe_state_mark(pe, EEH_PE_RECOVERING); |
| 80 | pr_info("EEH: Detected PCI bus error on PHB#%d-PE#%x\n", |
| 81 | pe->phb->global_number, pe->addr); |
Linas Vepstas | 8c33fd11 | 2006-03-29 15:29:18 -0600 | [diff] [blame] | 82 | |
Andrew Morton | 9b218f6 | 2012-03-28 12:20:58 +0000 | [diff] [blame] | 83 | set_current_state(TASK_INTERRUPTIBLE); /* Don't add to load average */ |
Gavin Shan | 9b3c76f | 2012-09-07 22:44:19 +0000 | [diff] [blame] | 84 | eeh_handle_event(pe); |
Gavin Shan | 120dc49 | 2012-09-07 22:44:18 +0000 | [diff] [blame] | 85 | eeh_pe_state_clear(pe, EEH_PE_RECOVERING); |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 86 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 87 | kfree(event); |
| 88 | mutex_unlock(&eeh_event_mutex); |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 89 | |
Linas Vepstas | ac325ac | 2006-04-18 21:05:21 -0700 | [diff] [blame] | 90 | /* If there are no new errors after an hour, clear the counter. */ |
Gavin Shan | 120dc49 | 2012-09-07 22:44:18 +0000 | [diff] [blame] | 91 | if (pe && pe->freeze_count > 0) { |
Gavin Shan | 29f8bf1 | 2012-02-27 20:04:02 +0000 | [diff] [blame] | 92 | msleep_interruptible(3600*1000); |
Gavin Shan | 120dc49 | 2012-09-07 22:44:18 +0000 | [diff] [blame] | 93 | if (pe->freeze_count > 0) |
| 94 | pe->freeze_count--; |
Gavin Shan | 40a7cd9 | 2012-02-27 20:04:08 +0000 | [diff] [blame] | 95 | |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 96 | } |
| 97 | |
| 98 | return 0; |
| 99 | } |
| 100 | |
| 101 | /** |
Gavin Shan | 29f8bf1 | 2012-02-27 20:04:02 +0000 | [diff] [blame] | 102 | * eeh_thread_launcher - Start kernel thread to handle EEH events |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 103 | * @dummy - unused |
Gavin Shan | 29f8bf1 | 2012-02-27 20:04:02 +0000 | [diff] [blame] | 104 | * |
| 105 | * This routine is called to start the kernel thread for processing |
| 106 | * EEH event. |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 107 | */ |
David Howells | c402895 | 2006-11-22 14:57:56 +0000 | [diff] [blame] | 108 | static void eeh_thread_launcher(struct work_struct *dummy) |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 109 | { |
Al Viro | ecf89e5 | 2012-10-02 15:32:10 -0400 | [diff] [blame] | 110 | if (IS_ERR(kthread_run(eeh_event_handler, NULL, "eehd"))) |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 111 | printk(KERN_ERR "Failed to start EEH daemon\n"); |
| 112 | } |
| 113 | |
| 114 | /** |
Gavin Shan | 29f8bf1 | 2012-02-27 20:04:02 +0000 | [diff] [blame] | 115 | * eeh_send_failure_event - Generate a PCI error event |
Gavin Shan | c533b46 | 2012-09-07 22:44:11 +0000 | [diff] [blame] | 116 | * @pe: EEH PE |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 117 | * |
| 118 | * This routine can be called within an interrupt context; |
| 119 | * the actual event will be delivered in a normal context |
| 120 | * (from a workqueue). |
| 121 | */ |
Gavin Shan | c533b46 | 2012-09-07 22:44:11 +0000 | [diff] [blame] | 122 | int eeh_send_failure_event(struct eeh_pe *pe) |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 123 | { |
| 124 | unsigned long flags; |
| 125 | struct eeh_event *event; |
| 126 | |
Gavin Shan | 7e4bbaf | 2012-09-07 22:44:03 +0000 | [diff] [blame] | 127 | event = kzalloc(sizeof(*event), GFP_ATOMIC); |
Gavin Shan | c533b46 | 2012-09-07 22:44:11 +0000 | [diff] [blame] | 128 | if (!event) { |
| 129 | pr_err("EEH: out of memory, event not handled\n"); |
| 130 | return -ENOMEM; |
| 131 | } |
| 132 | event->pe = pe; |
Linas Vepstas | 172ca92 | 2005-11-03 18:50:04 -0600 | [diff] [blame] | 133 | |
| 134 | /* We may or may not be called in an interrupt context */ |
| 135 | spin_lock_irqsave(&eeh_eventlist_lock, flags); |
| 136 | list_add(&event->list, &eeh_eventlist); |
| 137 | spin_unlock_irqrestore(&eeh_eventlist_lock, flags); |
| 138 | |
| 139 | schedule_work(&eeh_event_wq); |
| 140 | |
| 141 | return 0; |
| 142 | } |