blob: 185bedd926df7258bef7070d97c68f577bf91298 [file] [log] [blame]
Linas Vepstas172ca922005-11-03 18:50:04 -06001/*
Linas Vepstas172ca922005-11-03 18:50:04 -06002 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License as published by
4 * the Free Software Foundation; either version 2 of the License, or
5 * (at your option) any later version.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 * You should have received a copy of the GNU General Public License
13 * along with this program; if not, write to the Free Software
14 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15 *
16 * Copyright (c) 2005 Linas Vepstas <linas@linas.org>
17 */
18
Linas Vepstasac325ac2006-04-18 21:05:21 -070019#include <linux/delay.h>
Linas Vepstas172ca922005-11-03 18:50:04 -060020#include <linux/list.h>
Linas Vepstas8c33fd112006-03-29 15:29:18 -060021#include <linux/mutex.h>
Paul Gortmaker62fe91b2011-05-27 14:25:11 -040022#include <linux/sched.h>
Linas Vepstas172ca922005-11-03 18:50:04 -060023#include <linux/pci.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090024#include <linux/slab.h>
Linas Vepstas8c33fd112006-03-29 15:29:18 -060025#include <linux/workqueue.h>
Al Viroecf89e52012-10-02 15:32:10 -040026#include <linux/kthread.h>
Linas Vepstas172ca922005-11-03 18:50:04 -060027#include <asm/eeh_event.h>
Linas Vepstas77bd7412005-11-03 18:52:49 -060028#include <asm/ppc-pci.h>
Linas Vepstas172ca922005-11-03 18:50:04 -060029
30/** Overview:
31 * EEH error states may be detected within exception handlers;
32 * however, the recovery processing needs to occur asynchronously
33 * in a normal kernel context and not an interrupt context.
34 * This pair of routines creates an event and queues it onto a
35 * work-queue, where a worker thread can drive recovery.
36 */
37
38/* EEH event workqueue setup. */
Ingo Molnar34af9462006-06-27 02:53:55 -070039static DEFINE_SPINLOCK(eeh_eventlist_lock);
Linas Vepstas172ca922005-11-03 18:50:04 -060040LIST_HEAD(eeh_eventlist);
David Howellsc4028952006-11-22 14:57:56 +000041static void eeh_thread_launcher(struct work_struct *);
42DECLARE_WORK(eeh_event_wq, eeh_thread_launcher);
Linas Vepstas172ca922005-11-03 18:50:04 -060043
Linas Vepstas8c33fd112006-03-29 15:29:18 -060044/* Serialize reset sequences for a given pci device */
45DEFINE_MUTEX(eeh_event_mutex);
46
Linas Vepstas172ca922005-11-03 18:50:04 -060047/**
Gavin Shan29f8bf12012-02-27 20:04:02 +000048 * eeh_event_handler - Dispatch EEH events.
Linas Vepstas172ca922005-11-03 18:50:04 -060049 * @dummy - unused
Linas Vepstas8c33fd112006-03-29 15:29:18 -060050 *
51 * The detection of a frozen slot can occur inside an interrupt,
52 * where it can be hard to do anything about it. The goal of this
53 * routine is to pull these detection events out of the context
54 * of the interrupt handler, and re-dispatch them for processing
55 * at a later time in a normal context.
Linas Vepstas172ca922005-11-03 18:50:04 -060056 */
57static int eeh_event_handler(void * dummy)
58{
59 unsigned long flags;
Gavin Shan40a7cd92012-02-27 20:04:08 +000060 struct eeh_event *event;
Gavin Shan120dc492012-09-07 22:44:18 +000061 struct eeh_pe *pe;
Linas Vepstas172ca922005-11-03 18:50:04 -060062
Linas Vepstasac325ac2006-04-18 21:05:21 -070063 spin_lock_irqsave(&eeh_eventlist_lock, flags);
64 event = NULL;
Linas Vepstas172ca922005-11-03 18:50:04 -060065
Linas Vepstasac325ac2006-04-18 21:05:21 -070066 /* Unqueue the event, get ready to process. */
67 if (!list_empty(&eeh_eventlist)) {
68 event = list_entry(eeh_eventlist.next, struct eeh_event, list);
69 list_del(&event->list);
70 }
71 spin_unlock_irqrestore(&eeh_eventlist_lock, flags);
Linas Vepstas77bd7412005-11-03 18:52:49 -060072
Linas Vepstasac325ac2006-04-18 21:05:21 -070073 if (event == NULL)
74 return 0;
Linas Vepstas8c33fd112006-03-29 15:29:18 -060075
Linas Vepstasac325ac2006-04-18 21:05:21 -070076 /* Serialize processing of EEH events */
77 mutex_lock(&eeh_event_mutex);
Gavin Shan120dc492012-09-07 22:44:18 +000078 pe = event->pe;
79 eeh_pe_state_mark(pe, EEH_PE_RECOVERING);
80 pr_info("EEH: Detected PCI bus error on PHB#%d-PE#%x\n",
81 pe->phb->global_number, pe->addr);
Linas Vepstas8c33fd112006-03-29 15:29:18 -060082
Andrew Morton9b218f62012-03-28 12:20:58 +000083 set_current_state(TASK_INTERRUPTIBLE); /* Don't add to load average */
Gavin Shan9b3c76f2012-09-07 22:44:19 +000084 eeh_handle_event(pe);
Gavin Shan120dc492012-09-07 22:44:18 +000085 eeh_pe_state_clear(pe, EEH_PE_RECOVERING);
Gavin Shan40a7cd92012-02-27 20:04:08 +000086
Linas Vepstasac325ac2006-04-18 21:05:21 -070087 kfree(event);
88 mutex_unlock(&eeh_event_mutex);
Linas Vepstas172ca922005-11-03 18:50:04 -060089
Linas Vepstasac325ac2006-04-18 21:05:21 -070090 /* If there are no new errors after an hour, clear the counter. */
Gavin Shan120dc492012-09-07 22:44:18 +000091 if (pe && pe->freeze_count > 0) {
Gavin Shan29f8bf12012-02-27 20:04:02 +000092 msleep_interruptible(3600*1000);
Gavin Shan120dc492012-09-07 22:44:18 +000093 if (pe->freeze_count > 0)
94 pe->freeze_count--;
Gavin Shan40a7cd92012-02-27 20:04:08 +000095
Linas Vepstas172ca922005-11-03 18:50:04 -060096 }
97
98 return 0;
99}
100
101/**
Gavin Shan29f8bf12012-02-27 20:04:02 +0000102 * eeh_thread_launcher - Start kernel thread to handle EEH events
Linas Vepstas172ca922005-11-03 18:50:04 -0600103 * @dummy - unused
Gavin Shan29f8bf12012-02-27 20:04:02 +0000104 *
105 * This routine is called to start the kernel thread for processing
106 * EEH event.
Linas Vepstas172ca922005-11-03 18:50:04 -0600107 */
David Howellsc4028952006-11-22 14:57:56 +0000108static void eeh_thread_launcher(struct work_struct *dummy)
Linas Vepstas172ca922005-11-03 18:50:04 -0600109{
Al Viroecf89e52012-10-02 15:32:10 -0400110 if (IS_ERR(kthread_run(eeh_event_handler, NULL, "eehd")))
Linas Vepstas172ca922005-11-03 18:50:04 -0600111 printk(KERN_ERR "Failed to start EEH daemon\n");
112}
113
114/**
Gavin Shan29f8bf12012-02-27 20:04:02 +0000115 * eeh_send_failure_event - Generate a PCI error event
Gavin Shanc533b462012-09-07 22:44:11 +0000116 * @pe: EEH PE
Linas Vepstas172ca922005-11-03 18:50:04 -0600117 *
118 * This routine can be called within an interrupt context;
119 * the actual event will be delivered in a normal context
120 * (from a workqueue).
121 */
Gavin Shanc533b462012-09-07 22:44:11 +0000122int eeh_send_failure_event(struct eeh_pe *pe)
Linas Vepstas172ca922005-11-03 18:50:04 -0600123{
124 unsigned long flags;
125 struct eeh_event *event;
126
Gavin Shan7e4bbaf2012-09-07 22:44:03 +0000127 event = kzalloc(sizeof(*event), GFP_ATOMIC);
Gavin Shanc533b462012-09-07 22:44:11 +0000128 if (!event) {
129 pr_err("EEH: out of memory, event not handled\n");
130 return -ENOMEM;
131 }
132 event->pe = pe;
Linas Vepstas172ca922005-11-03 18:50:04 -0600133
134 /* We may or may not be called in an interrupt context */
135 spin_lock_irqsave(&eeh_eventlist_lock, flags);
136 list_add(&event->list, &eeh_eventlist);
137 spin_unlock_irqrestore(&eeh_eventlist_lock, flags);
138
139 schedule_work(&eeh_event_wq);
140
141 return 0;
142}