Petr Mladek | 42a0bb3 | 2016-05-20 17:00:33 -0700 | [diff] [blame] | 1 | /* |
| 2 | * nmi.c - Safe printk in NMI context |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or |
| 5 | * modify it under the terms of the GNU General Public License |
| 6 | * as published by the Free Software Foundation; either version 2 |
| 7 | * of the License, or (at your option) any later version. |
| 8 | * |
| 9 | * This program is distributed in the hope that it will be useful, |
| 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 12 | * GNU General Public License for more details. |
| 13 | * |
| 14 | * You should have received a copy of the GNU General Public License |
| 15 | * along with this program; if not, see <http://www.gnu.org/licenses/>. |
| 16 | */ |
| 17 | |
| 18 | #include <linux/preempt.h> |
| 19 | #include <linux/spinlock.h> |
Petr Mladek | cf9b110 | 2016-05-20 17:00:42 -0700 | [diff] [blame] | 20 | #include <linux/debug_locks.h> |
Petr Mladek | 42a0bb3 | 2016-05-20 17:00:33 -0700 | [diff] [blame] | 21 | #include <linux/smp.h> |
| 22 | #include <linux/cpumask.h> |
| 23 | #include <linux/irq_work.h> |
| 24 | #include <linux/printk.h> |
| 25 | |
| 26 | #include "internal.h" |
| 27 | |
| 28 | /* |
| 29 | * printk() could not take logbuf_lock in NMI context. Instead, |
| 30 | * it uses an alternative implementation that temporary stores |
| 31 | * the strings into a per-CPU buffer. The content of the buffer |
| 32 | * is later flushed into the main ring buffer via IRQ work. |
| 33 | * |
| 34 | * The alternative implementation is chosen transparently |
| 35 | * via @printk_func per-CPU variable. |
| 36 | * |
| 37 | * The implementation allows to flush the strings also from another CPU. |
| 38 | * There are situations when we want to make sure that all buffers |
| 39 | * were handled or when IRQs are blocked. |
| 40 | */ |
| 41 | DEFINE_PER_CPU(printk_func_t, printk_func) = vprintk_default; |
| 42 | static int printk_nmi_irq_ready; |
Petr Mladek | b522dea | 2016-05-20 17:00:36 -0700 | [diff] [blame] | 43 | atomic_t nmi_message_lost; |
Petr Mladek | 42a0bb3 | 2016-05-20 17:00:33 -0700 | [diff] [blame] | 44 | |
Petr Mladek | 427934b | 2016-05-20 17:00:39 -0700 | [diff] [blame] | 45 | #define NMI_LOG_BUF_LEN ((1 << CONFIG_NMI_LOG_BUF_SHIFT) - \ |
| 46 | sizeof(atomic_t) - sizeof(struct irq_work)) |
Petr Mladek | 42a0bb3 | 2016-05-20 17:00:33 -0700 | [diff] [blame] | 47 | |
| 48 | struct nmi_seq_buf { |
| 49 | atomic_t len; /* length of written data */ |
| 50 | struct irq_work work; /* IRQ work that flushes the buffer */ |
| 51 | unsigned char buffer[NMI_LOG_BUF_LEN]; |
| 52 | }; |
| 53 | static DEFINE_PER_CPU(struct nmi_seq_buf, nmi_print_seq); |
| 54 | |
| 55 | /* |
| 56 | * Safe printk() for NMI context. It uses a per-CPU buffer to |
| 57 | * store the message. NMIs are not nested, so there is always only |
| 58 | * one writer running. But the buffer might get flushed from another |
| 59 | * CPU, so we need to be careful. |
| 60 | */ |
Linus Torvalds | a0cba21 | 2016-08-09 10:48:18 -0700 | [diff] [blame] | 61 | static int vprintk_nmi(const char *fmt, va_list args) |
Petr Mladek | 42a0bb3 | 2016-05-20 17:00:33 -0700 | [diff] [blame] | 62 | { |
| 63 | struct nmi_seq_buf *s = this_cpu_ptr(&nmi_print_seq); |
| 64 | int add = 0; |
| 65 | size_t len; |
Tetsuo Handa | db2baee | 2018-05-11 19:54:19 +0900 | [diff] [blame] | 66 | va_list ap; |
Petr Mladek | 42a0bb3 | 2016-05-20 17:00:33 -0700 | [diff] [blame] | 67 | |
| 68 | again: |
| 69 | len = atomic_read(&s->len); |
| 70 | |
Petr Mladek | b522dea | 2016-05-20 17:00:36 -0700 | [diff] [blame] | 71 | if (len >= sizeof(s->buffer)) { |
| 72 | atomic_inc(&nmi_message_lost); |
Petr Mladek | 42a0bb3 | 2016-05-20 17:00:33 -0700 | [diff] [blame] | 73 | return 0; |
Petr Mladek | b522dea | 2016-05-20 17:00:36 -0700 | [diff] [blame] | 74 | } |
Petr Mladek | 42a0bb3 | 2016-05-20 17:00:33 -0700 | [diff] [blame] | 75 | |
| 76 | /* |
| 77 | * Make sure that all old data have been read before the buffer was |
| 78 | * reseted. This is not needed when we just append data. |
| 79 | */ |
| 80 | if (!len) |
| 81 | smp_rmb(); |
| 82 | |
Tetsuo Handa | db2baee | 2018-05-11 19:54:19 +0900 | [diff] [blame] | 83 | va_copy(ap, args); |
| 84 | add = vsnprintf(s->buffer + len, sizeof(s->buffer) - len, fmt, ap); |
| 85 | va_end(ap); |
Petr Mladek | 42a0bb3 | 2016-05-20 17:00:33 -0700 | [diff] [blame] | 86 | |
| 87 | /* |
| 88 | * Do it once again if the buffer has been flushed in the meantime. |
| 89 | * Note that atomic_cmpxchg() is an implicit memory barrier that |
| 90 | * makes sure that the data were written before updating s->len. |
| 91 | */ |
| 92 | if (atomic_cmpxchg(&s->len, len, len + add) != len) |
| 93 | goto again; |
| 94 | |
| 95 | /* Get flushed in a more safe context. */ |
| 96 | if (add && printk_nmi_irq_ready) { |
| 97 | /* Make sure that IRQ work is really initialized. */ |
| 98 | smp_rmb(); |
| 99 | irq_work_queue(&s->work); |
| 100 | } |
| 101 | |
| 102 | return add; |
| 103 | } |
| 104 | |
Sergey Senozhatsky | 19feeff | 2016-09-01 16:15:04 -0700 | [diff] [blame] | 105 | static void printk_nmi_flush_line(const char *text, int len) |
Petr Mladek | 42a0bb3 | 2016-05-20 17:00:33 -0700 | [diff] [blame] | 106 | { |
Petr Mladek | cf9b110 | 2016-05-20 17:00:42 -0700 | [diff] [blame] | 107 | /* |
| 108 | * The buffers are flushed in NMI only on panic. The messages must |
| 109 | * go only into the ring buffer at this stage. Consoles will get |
| 110 | * explicitly called later when a crashdump is not generated. |
| 111 | */ |
| 112 | if (in_nmi()) |
Sergey Senozhatsky | 19feeff | 2016-09-01 16:15:04 -0700 | [diff] [blame] | 113 | printk_deferred("%.*s", len, text); |
Petr Mladek | cf9b110 | 2016-05-20 17:00:42 -0700 | [diff] [blame] | 114 | else |
Sergey Senozhatsky | 19feeff | 2016-09-01 16:15:04 -0700 | [diff] [blame] | 115 | printk("%.*s", len, text); |
Petr Mladek | cf9b110 | 2016-05-20 17:00:42 -0700 | [diff] [blame] | 116 | |
Petr Mladek | 42a0bb3 | 2016-05-20 17:00:33 -0700 | [diff] [blame] | 117 | } |
| 118 | |
| 119 | /* |
Sergey Senozhatsky | 19feeff | 2016-09-01 16:15:04 -0700 | [diff] [blame] | 120 | * printk one line from the temporary buffer from @start index until |
| 121 | * and including the @end index. |
| 122 | */ |
| 123 | static void printk_nmi_flush_seq_line(struct nmi_seq_buf *s, |
| 124 | int start, int end) |
| 125 | { |
| 126 | const char *buf = s->buffer + start; |
| 127 | |
| 128 | printk_nmi_flush_line(buf, (end - start) + 1); |
| 129 | } |
| 130 | |
| 131 | /* |
Petr Mladek | 42a0bb3 | 2016-05-20 17:00:33 -0700 | [diff] [blame] | 132 | * Flush data from the associated per_CPU buffer. The function |
| 133 | * can be called either via IRQ work or independently. |
| 134 | */ |
| 135 | static void __printk_nmi_flush(struct irq_work *work) |
| 136 | { |
| 137 | static raw_spinlock_t read_lock = |
| 138 | __RAW_SPIN_LOCK_INITIALIZER(read_lock); |
| 139 | struct nmi_seq_buf *s = container_of(work, struct nmi_seq_buf, work); |
| 140 | unsigned long flags; |
| 141 | size_t len, size; |
| 142 | int i, last_i; |
| 143 | |
| 144 | /* |
| 145 | * The lock has two functions. First, one reader has to flush all |
| 146 | * available message to make the lockless synchronization with |
| 147 | * writers easier. Second, we do not want to mix messages from |
| 148 | * different CPUs. This is especially important when printing |
| 149 | * a backtrace. |
| 150 | */ |
| 151 | raw_spin_lock_irqsave(&read_lock, flags); |
| 152 | |
| 153 | i = 0; |
| 154 | more: |
| 155 | len = atomic_read(&s->len); |
| 156 | |
| 157 | /* |
| 158 | * This is just a paranoid check that nobody has manipulated |
| 159 | * the buffer an unexpected way. If we printed something then |
| 160 | * @len must only increase. |
| 161 | */ |
Sergey Senozhatsky | 19feeff | 2016-09-01 16:15:04 -0700 | [diff] [blame] | 162 | if (i && i >= len) { |
| 163 | const char *msg = "printk_nmi_flush: internal error\n"; |
| 164 | |
| 165 | printk_nmi_flush_line(msg, strlen(msg)); |
| 166 | } |
Petr Mladek | 42a0bb3 | 2016-05-20 17:00:33 -0700 | [diff] [blame] | 167 | |
| 168 | if (!len) |
| 169 | goto out; /* Someone else has already flushed the buffer. */ |
| 170 | |
| 171 | /* Make sure that data has been written up to the @len */ |
| 172 | smp_rmb(); |
| 173 | |
| 174 | size = min(len, sizeof(s->buffer)); |
| 175 | last_i = i; |
| 176 | |
| 177 | /* Print line by line. */ |
| 178 | for (; i < size; i++) { |
| 179 | if (s->buffer[i] == '\n') { |
Sergey Senozhatsky | 19feeff | 2016-09-01 16:15:04 -0700 | [diff] [blame] | 180 | printk_nmi_flush_seq_line(s, last_i, i); |
Petr Mladek | 42a0bb3 | 2016-05-20 17:00:33 -0700 | [diff] [blame] | 181 | last_i = i + 1; |
| 182 | } |
| 183 | } |
| 184 | /* Check if there was a partial line. */ |
| 185 | if (last_i < size) { |
Sergey Senozhatsky | 19feeff | 2016-09-01 16:15:04 -0700 | [diff] [blame] | 186 | printk_nmi_flush_seq_line(s, last_i, size - 1); |
| 187 | printk_nmi_flush_line("\n", strlen("\n")); |
Petr Mladek | 42a0bb3 | 2016-05-20 17:00:33 -0700 | [diff] [blame] | 188 | } |
| 189 | |
| 190 | /* |
| 191 | * Check that nothing has got added in the meantime and truncate |
| 192 | * the buffer. Note that atomic_cmpxchg() is an implicit memory |
| 193 | * barrier that makes sure that the data were copied before |
| 194 | * updating s->len. |
| 195 | */ |
| 196 | if (atomic_cmpxchg(&s->len, len, 0) != len) |
| 197 | goto more; |
| 198 | |
| 199 | out: |
| 200 | raw_spin_unlock_irqrestore(&read_lock, flags); |
| 201 | } |
| 202 | |
| 203 | /** |
| 204 | * printk_nmi_flush - flush all per-cpu nmi buffers. |
| 205 | * |
| 206 | * The buffers are flushed automatically via IRQ work. This function |
| 207 | * is useful only when someone wants to be sure that all buffers have |
| 208 | * been flushed at some point. |
| 209 | */ |
| 210 | void printk_nmi_flush(void) |
| 211 | { |
| 212 | int cpu; |
| 213 | |
| 214 | for_each_possible_cpu(cpu) |
| 215 | __printk_nmi_flush(&per_cpu(nmi_print_seq, cpu).work); |
| 216 | } |
| 217 | |
Petr Mladek | cf9b110 | 2016-05-20 17:00:42 -0700 | [diff] [blame] | 218 | /** |
| 219 | * printk_nmi_flush_on_panic - flush all per-cpu nmi buffers when the system |
| 220 | * goes down. |
| 221 | * |
| 222 | * Similar to printk_nmi_flush() but it can be called even in NMI context when |
| 223 | * the system goes down. It does the best effort to get NMI messages into |
| 224 | * the main ring buffer. |
| 225 | * |
| 226 | * Note that it could try harder when there is only one CPU online. |
| 227 | */ |
| 228 | void printk_nmi_flush_on_panic(void) |
| 229 | { |
| 230 | /* |
| 231 | * Make sure that we could access the main ring buffer. |
| 232 | * Do not risk a double release when more CPUs are up. |
| 233 | */ |
| 234 | if (in_nmi() && raw_spin_is_locked(&logbuf_lock)) { |
| 235 | if (num_online_cpus() > 1) |
| 236 | return; |
| 237 | |
| 238 | debug_locks_off(); |
| 239 | raw_spin_lock_init(&logbuf_lock); |
| 240 | } |
| 241 | |
| 242 | printk_nmi_flush(); |
| 243 | } |
| 244 | |
Petr Mladek | 42a0bb3 | 2016-05-20 17:00:33 -0700 | [diff] [blame] | 245 | void __init printk_nmi_init(void) |
| 246 | { |
| 247 | int cpu; |
| 248 | |
| 249 | for_each_possible_cpu(cpu) { |
| 250 | struct nmi_seq_buf *s = &per_cpu(nmi_print_seq, cpu); |
| 251 | |
| 252 | init_irq_work(&s->work, __printk_nmi_flush); |
| 253 | } |
| 254 | |
| 255 | /* Make sure that IRQ works are initialized before enabling. */ |
| 256 | smp_wmb(); |
| 257 | printk_nmi_irq_ready = 1; |
| 258 | |
| 259 | /* Flush pending messages that did not have scheduled IRQ works. */ |
| 260 | printk_nmi_flush(); |
| 261 | } |
| 262 | |
Steven Rostedt (VMware) | 6c6d174 | 2018-09-05 16:29:49 -0400 | [diff] [blame] | 263 | void notrace printk_nmi_enter(void) |
Petr Mladek | 42a0bb3 | 2016-05-20 17:00:33 -0700 | [diff] [blame] | 264 | { |
| 265 | this_cpu_write(printk_func, vprintk_nmi); |
| 266 | } |
| 267 | |
Steven Rostedt (VMware) | 6c6d174 | 2018-09-05 16:29:49 -0400 | [diff] [blame] | 268 | void notrace printk_nmi_exit(void) |
Petr Mladek | 42a0bb3 | 2016-05-20 17:00:33 -0700 | [diff] [blame] | 269 | { |
| 270 | this_cpu_write(printk_func, vprintk_default); |
| 271 | } |