Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 1 | /* |
| 2 | * Code for replacing ftrace calls with jumps. |
| 3 | * |
| 4 | * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> |
Wu Zhangjin | e424054 | 2010-05-14 19:08:32 +0800 | [diff] [blame] | 5 | * Copyright (C) 2009, 2010 DSLab, Lanzhou University, China |
Wu Zhangjin | f7a904d | 2010-01-04 17:16:51 +0800 | [diff] [blame] | 6 | * Author: Wu Zhangjin <wuzhangjin@gmail.com> |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 7 | * |
| 8 | * Thanks goes to Steven Rostedt for writing the original x86 version. |
| 9 | */ |
| 10 | |
| 11 | #include <linux/uaccess.h> |
| 12 | #include <linux/init.h> |
| 13 | #include <linux/ftrace.h> |
Ralf Baechle | 19e2e17 | 2012-07-13 23:38:17 +0200 | [diff] [blame] | 14 | #include <linux/syscalls.h> |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 15 | |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 16 | #include <asm/asm.h> |
| 17 | #include <asm/asm-offsets.h> |
Wu Zhangjin | e424054 | 2010-05-14 19:08:32 +0800 | [diff] [blame] | 18 | #include <asm/cacheflush.h> |
Ralf Baechle | 19e2e17 | 2012-07-13 23:38:17 +0200 | [diff] [blame] | 19 | #include <asm/syscall.h> |
Wu Zhangjin | e424054 | 2010-05-14 19:08:32 +0800 | [diff] [blame] | 20 | #include <asm/uasm.h> |
Ralf Baechle | 19e2e17 | 2012-07-13 23:38:17 +0200 | [diff] [blame] | 21 | #include <asm/unistd.h> |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 22 | |
Wu Zhangjin | d9cdb2f | 2011-01-20 03:28:29 +0800 | [diff] [blame] | 23 | #include <asm-generic/sections.h> |
Wu Zhangjin | c9f8487 | 2010-05-14 19:08:34 +0800 | [diff] [blame] | 24 | |
Thomas Gleixner | 49de830 | 2011-07-23 12:41:23 +0000 | [diff] [blame] | 25 | #if defined(KBUILD_MCOUNT_RA_ADDRESS) && defined(CONFIG_32BIT) |
| 26 | #define MCOUNT_OFFSET_INSNS 5 |
| 27 | #else |
| 28 | #define MCOUNT_OFFSET_INSNS 4 |
| 29 | #endif |
| 30 | |
Markos Chandras | cb2f993 | 2013-06-10 10:35:26 +0000 | [diff] [blame] | 31 | #ifdef CONFIG_DYNAMIC_FTRACE |
| 32 | |
Al Cooper | 58b6940 | 2013-01-16 22:43:28 +0000 | [diff] [blame] | 33 | /* Arch override because MIPS doesn't need to run this from stop_machine() */ |
| 34 | void arch_ftrace_update_code(int command) |
| 35 | { |
| 36 | ftrace_modify_all_code(command); |
| 37 | } |
| 38 | |
Markos Chandras | cb2f993 | 2013-06-10 10:35:26 +0000 | [diff] [blame] | 39 | #endif |
| 40 | |
Thomas Gleixner | 49de830 | 2011-07-23 12:41:23 +0000 | [diff] [blame] | 41 | /* |
| 42 | * Check if the address is in kernel space |
| 43 | * |
| 44 | * Clone core_kernel_text() from kernel/extable.c, but doesn't call |
| 45 | * init_kernel_text() for Ftrace doesn't trace functions in init sections. |
| 46 | */ |
| 47 | static inline int in_kernel_space(unsigned long ip) |
| 48 | { |
| 49 | if (ip >= (unsigned long)_stext && |
| 50 | ip <= (unsigned long)_etext) |
| 51 | return 1; |
| 52 | return 0; |
| 53 | } |
| 54 | |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 55 | #ifdef CONFIG_DYNAMIC_FTRACE |
| 56 | |
| 57 | #define JAL 0x0c000000 /* jump & link: ip --> ra, jump to target */ |
| 58 | #define ADDR_MASK 0x03ffffff /* op_code|addr : 31...26|25 ....0 */ |
David Daney | c54794d | 2010-12-28 13:21:37 -0800 | [diff] [blame] | 59 | #define JUMP_RANGE_MASK ((1UL << 28) - 1) |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 60 | |
Wu Zhangjin | 4d6829f | 2010-05-14 19:08:31 +0800 | [diff] [blame] | 61 | #define INSN_NOP 0x00000000 /* nop */ |
Wu Zhangjin | e424054 | 2010-05-14 19:08:32 +0800 | [diff] [blame] | 62 | #define INSN_JAL(addr) \ |
| 63 | ((unsigned int)(JAL | (((addr) >> 2) & ADDR_MASK))) |
| 64 | |
| 65 | static unsigned int insn_jal_ftrace_caller __read_mostly; |
Petri Gynther | dce0e7d | 2014-07-23 22:55:02 -0700 | [diff] [blame] | 66 | static unsigned int insn_la_mcount[2] __read_mostly; |
Wu Zhangjin | e424054 | 2010-05-14 19:08:32 +0800 | [diff] [blame] | 67 | static unsigned int insn_j_ftrace_graph_caller __maybe_unused __read_mostly; |
| 68 | |
| 69 | static inline void ftrace_dyn_arch_init_insns(void) |
| 70 | { |
| 71 | u32 *buf; |
| 72 | unsigned int v1; |
| 73 | |
Petri Gynther | dce0e7d | 2014-07-23 22:55:02 -0700 | [diff] [blame] | 74 | /* la v1, _mcount */ |
Wu Zhangjin | e424054 | 2010-05-14 19:08:32 +0800 | [diff] [blame] | 75 | v1 = 3; |
Petri Gynther | dce0e7d | 2014-07-23 22:55:02 -0700 | [diff] [blame] | 76 | buf = (u32 *)&insn_la_mcount[0]; |
| 77 | UASM_i_LA(&buf, v1, MCOUNT_ADDR); |
Wu Zhangjin | e424054 | 2010-05-14 19:08:32 +0800 | [diff] [blame] | 78 | |
| 79 | /* jal (ftrace_caller + 8), jump over the first two instruction */ |
| 80 | buf = (u32 *)&insn_jal_ftrace_caller; |
David Daney | c54794d | 2010-12-28 13:21:37 -0800 | [diff] [blame] | 81 | uasm_i_jal(&buf, (FTRACE_ADDR + 8) & JUMP_RANGE_MASK); |
Wu Zhangjin | e424054 | 2010-05-14 19:08:32 +0800 | [diff] [blame] | 82 | |
| 83 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 84 | /* j ftrace_graph_caller */ |
| 85 | buf = (u32 *)&insn_j_ftrace_graph_caller; |
David Daney | c54794d | 2010-12-28 13:21:37 -0800 | [diff] [blame] | 86 | uasm_i_j(&buf, (unsigned long)ftrace_graph_caller & JUMP_RANGE_MASK); |
Wu Zhangjin | e424054 | 2010-05-14 19:08:32 +0800 | [diff] [blame] | 87 | #endif |
| 88 | } |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 89 | |
| 90 | static int ftrace_modify_code(unsigned long ip, unsigned int new_code) |
| 91 | { |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 92 | int faulted; |
Leonid Yegoshin | 6ebda44 | 2013-12-16 12:06:55 +0000 | [diff] [blame] | 93 | mm_segment_t old_fs; |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 94 | |
| 95 | /* *(unsigned int *)ip = new_code; */ |
| 96 | safe_store_code(new_code, ip, faulted); |
| 97 | |
| 98 | if (unlikely(faulted)) |
| 99 | return -EFAULT; |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 100 | |
Leonid Yegoshin | 6ebda44 | 2013-12-16 12:06:55 +0000 | [diff] [blame] | 101 | old_fs = get_fs(); |
| 102 | set_fs(get_ds()); |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 103 | flush_icache_range(ip, ip + 8); |
Leonid Yegoshin | 6ebda44 | 2013-12-16 12:06:55 +0000 | [diff] [blame] | 104 | set_fs(old_fs); |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 105 | |
| 106 | return 0; |
| 107 | } |
| 108 | |
Al Cooper | 58b6940 | 2013-01-16 22:43:28 +0000 | [diff] [blame] | 109 | #ifndef CONFIG_64BIT |
| 110 | static int ftrace_modify_code_2(unsigned long ip, unsigned int new_code1, |
| 111 | unsigned int new_code2) |
| 112 | { |
| 113 | int faulted; |
Petri Gynther | dce0e7d | 2014-07-23 22:55:02 -0700 | [diff] [blame] | 114 | mm_segment_t old_fs; |
Al Cooper | 58b6940 | 2013-01-16 22:43:28 +0000 | [diff] [blame] | 115 | |
| 116 | safe_store_code(new_code1, ip, faulted); |
| 117 | if (unlikely(faulted)) |
| 118 | return -EFAULT; |
Petri Gynther | dce0e7d | 2014-07-23 22:55:02 -0700 | [diff] [blame] | 119 | |
| 120 | ip += 4; |
| 121 | safe_store_code(new_code2, ip, faulted); |
Al Cooper | 58b6940 | 2013-01-16 22:43:28 +0000 | [diff] [blame] | 122 | if (unlikely(faulted)) |
| 123 | return -EFAULT; |
Petri Gynther | dce0e7d | 2014-07-23 22:55:02 -0700 | [diff] [blame] | 124 | |
| 125 | ip -= 4; |
| 126 | old_fs = get_fs(); |
| 127 | set_fs(get_ds()); |
Viller Hsiao | a467109 | 2014-02-22 15:46:49 +0800 | [diff] [blame] | 128 | flush_icache_range(ip, ip + 8); |
Petri Gynther | dce0e7d | 2014-07-23 22:55:02 -0700 | [diff] [blame] | 129 | set_fs(old_fs); |
| 130 | |
| 131 | return 0; |
| 132 | } |
| 133 | |
| 134 | static int ftrace_modify_code_2r(unsigned long ip, unsigned int new_code1, |
| 135 | unsigned int new_code2) |
| 136 | { |
| 137 | int faulted; |
| 138 | mm_segment_t old_fs; |
| 139 | |
| 140 | ip += 4; |
| 141 | safe_store_code(new_code2, ip, faulted); |
| 142 | if (unlikely(faulted)) |
| 143 | return -EFAULT; |
| 144 | |
| 145 | ip -= 4; |
| 146 | safe_store_code(new_code1, ip, faulted); |
| 147 | if (unlikely(faulted)) |
| 148 | return -EFAULT; |
| 149 | |
| 150 | old_fs = get_fs(); |
| 151 | set_fs(get_ds()); |
| 152 | flush_icache_range(ip, ip + 8); |
| 153 | set_fs(old_fs); |
| 154 | |
Al Cooper | 58b6940 | 2013-01-16 22:43:28 +0000 | [diff] [blame] | 155 | return 0; |
| 156 | } |
| 157 | #endif |
| 158 | |
Wu Zhangjin | 7f21a60 | 2011-01-20 03:28:31 +0800 | [diff] [blame] | 159 | /* |
| 160 | * The details about the calling site of mcount on MIPS |
| 161 | * |
| 162 | * 1. For kernel: |
| 163 | * |
| 164 | * move at, ra |
| 165 | * jal _mcount --> nop |
Petri Gynther | dce0e7d | 2014-07-23 22:55:02 -0700 | [diff] [blame] | 166 | * sub sp, sp, 8 --> nop (CONFIG_32BIT) |
Wu Zhangjin | 7f21a60 | 2011-01-20 03:28:31 +0800 | [diff] [blame] | 167 | * |
| 168 | * 2. For modules: |
| 169 | * |
| 170 | * 2.1 For KBUILD_MCOUNT_RA_ADDRESS and CONFIG_32BIT |
| 171 | * |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 172 | * lui v1, hi_16bit_of_mcount --> b 1f (0x10000005) |
Petri Gynther | dce0e7d | 2014-07-23 22:55:02 -0700 | [diff] [blame] | 173 | * addiu v1, v1, low_16bit_of_mcount --> nop (CONFIG_32BIT) |
Wu Zhangjin | 7f21a60 | 2011-01-20 03:28:31 +0800 | [diff] [blame] | 174 | * move at, ra |
| 175 | * move $12, ra_address |
| 176 | * jalr v1 |
| 177 | * sub sp, sp, 8 |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 178 | * 1: offset = 5 instructions |
Wu Zhangjin | 7f21a60 | 2011-01-20 03:28:31 +0800 | [diff] [blame] | 179 | * 2.2 For the Other situations |
| 180 | * |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 181 | * lui v1, hi_16bit_of_mcount --> b 1f (0x10000004) |
Petri Gynther | dce0e7d | 2014-07-23 22:55:02 -0700 | [diff] [blame] | 182 | * addiu v1, v1, low_16bit_of_mcount --> nop (CONFIG_32BIT) |
Wu Zhangjin | 7f21a60 | 2011-01-20 03:28:31 +0800 | [diff] [blame] | 183 | * move at, ra |
| 184 | * jalr v1 |
| 185 | * nop | move $12, ra_address | sub sp, sp, 8 |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 186 | * 1: offset = 4 instructions |
Wu Zhangjin | 7f21a60 | 2011-01-20 03:28:31 +0800 | [diff] [blame] | 187 | */ |
| 188 | |
Wu Zhangjin | 7f21a60 | 2011-01-20 03:28:31 +0800 | [diff] [blame] | 189 | #define INSN_B_1F (0x10000000 | MCOUNT_OFFSET_INSNS) |
| 190 | |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 191 | int ftrace_make_nop(struct module *mod, |
| 192 | struct dyn_ftrace *rec, unsigned long addr) |
| 193 | { |
| 194 | unsigned int new; |
| 195 | unsigned long ip = rec->ip; |
| 196 | |
Wu Zhangjin | e424054 | 2010-05-14 19:08:32 +0800 | [diff] [blame] | 197 | /* |
Wu Zhangjin | d9cdb2f | 2011-01-20 03:28:29 +0800 | [diff] [blame] | 198 | * If ip is in kernel space, no long call, otherwise, long call is |
| 199 | * needed. |
Wu Zhangjin | e424054 | 2010-05-14 19:08:32 +0800 | [diff] [blame] | 200 | */ |
Wu Zhangjin | 7f21a60 | 2011-01-20 03:28:31 +0800 | [diff] [blame] | 201 | new = in_kernel_space(ip) ? INSN_NOP : INSN_B_1F; |
Al Cooper | 58b6940 | 2013-01-16 22:43:28 +0000 | [diff] [blame] | 202 | #ifdef CONFIG_64BIT |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 203 | return ftrace_modify_code(ip, new); |
Al Cooper | 58b6940 | 2013-01-16 22:43:28 +0000 | [diff] [blame] | 204 | #else |
| 205 | /* |
| 206 | * On 32 bit MIPS platforms, gcc adds a stack adjust |
| 207 | * instruction in the delay slot after the branch to |
| 208 | * mcount and expects mcount to restore the sp on return. |
| 209 | * This is based on a legacy API and does nothing but |
| 210 | * waste instructions so it's being removed at runtime. |
| 211 | */ |
| 212 | return ftrace_modify_code_2(ip, new, INSN_NOP); |
| 213 | #endif |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 214 | } |
| 215 | |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 216 | int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) |
| 217 | { |
| 218 | unsigned int new; |
| 219 | unsigned long ip = rec->ip; |
| 220 | |
Petri Gynther | dce0e7d | 2014-07-23 22:55:02 -0700 | [diff] [blame] | 221 | new = in_kernel_space(ip) ? insn_jal_ftrace_caller : insn_la_mcount[0]; |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 222 | |
Petri Gynther | dce0e7d | 2014-07-23 22:55:02 -0700 | [diff] [blame] | 223 | #ifdef CONFIG_64BIT |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 224 | return ftrace_modify_code(ip, new); |
Petri Gynther | dce0e7d | 2014-07-23 22:55:02 -0700 | [diff] [blame] | 225 | #else |
| 226 | return ftrace_modify_code_2r(ip, new, in_kernel_space(ip) ? |
| 227 | INSN_NOP : insn_la_mcount[1]); |
| 228 | #endif |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 229 | } |
| 230 | |
| 231 | #define FTRACE_CALL_IP ((unsigned long)(&ftrace_call)) |
| 232 | |
| 233 | int ftrace_update_ftrace_func(ftrace_func_t func) |
| 234 | { |
| 235 | unsigned int new; |
| 236 | |
Wu Zhangjin | 4d6829f | 2010-05-14 19:08:31 +0800 | [diff] [blame] | 237 | new = INSN_JAL((unsigned long)func); |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 238 | |
| 239 | return ftrace_modify_code(FTRACE_CALL_IP, new); |
| 240 | } |
| 241 | |
Jiri Slaby | 3a36cb1 | 2014-02-24 19:59:59 +0100 | [diff] [blame] | 242 | int __init ftrace_dyn_arch_init(void) |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 243 | { |
Wu Zhangjin | e424054 | 2010-05-14 19:08:32 +0800 | [diff] [blame] | 244 | /* Encode the instructions when booting */ |
| 245 | ftrace_dyn_arch_init_insns(); |
| 246 | |
| 247 | /* Remove "b ftrace_stub" to ensure ftrace_caller() is executed */ |
| 248 | ftrace_modify_code(MCOUNT_ADDR, INSN_NOP); |
| 249 | |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 250 | return 0; |
| 251 | } |
Wu Zhangjin | 68ccf75 | 2010-05-14 19:08:33 +0800 | [diff] [blame] | 252 | #endif /* CONFIG_DYNAMIC_FTRACE */ |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 253 | |
| 254 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 255 | |
Wu Zhangjin | e17ff5f | 2009-11-20 20:34:35 +0800 | [diff] [blame] | 256 | #ifdef CONFIG_DYNAMIC_FTRACE |
| 257 | |
| 258 | extern void ftrace_graph_call(void); |
Wu Zhangjin | e17ff5f | 2009-11-20 20:34:35 +0800 | [diff] [blame] | 259 | #define FTRACE_GRAPH_CALL_IP ((unsigned long)(&ftrace_graph_call)) |
| 260 | |
| 261 | int ftrace_enable_ftrace_graph_caller(void) |
| 262 | { |
| 263 | return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, |
Wu Zhangjin | e424054 | 2010-05-14 19:08:32 +0800 | [diff] [blame] | 264 | insn_j_ftrace_graph_caller); |
Wu Zhangjin | e17ff5f | 2009-11-20 20:34:35 +0800 | [diff] [blame] | 265 | } |
| 266 | |
| 267 | int ftrace_disable_ftrace_graph_caller(void) |
| 268 | { |
Wu Zhangjin | 4d6829f | 2010-05-14 19:08:31 +0800 | [diff] [blame] | 269 | return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, INSN_NOP); |
Wu Zhangjin | e17ff5f | 2009-11-20 20:34:35 +0800 | [diff] [blame] | 270 | } |
| 271 | |
Wu Zhangjin | 68ccf75 | 2010-05-14 19:08:33 +0800 | [diff] [blame] | 272 | #endif /* CONFIG_DYNAMIC_FTRACE */ |
Wu Zhangjin | e17ff5f | 2009-11-20 20:34:35 +0800 | [diff] [blame] | 273 | |
Wu Zhangjin | 7326c4e | 2009-11-20 20:34:38 +0800 | [diff] [blame] | 274 | #ifndef KBUILD_MCOUNT_RA_ADDRESS |
Wu Zhangjin | 68ccf75 | 2010-05-14 19:08:33 +0800 | [diff] [blame] | 275 | |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 276 | #define S_RA_SP (0xafbf << 16) /* s{d,w} ra, offset(sp) */ |
| 277 | #define S_R_SP (0xafb0 << 16) /* s{d,w} R, offset(sp) */ |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 278 | #define OFFSET_MASK 0xffff /* stack offset range: 0 ~ PT_SIZE */ |
| 279 | |
Wu Zhangjin | 2816e32 | 2011-01-20 03:28:30 +0800 | [diff] [blame] | 280 | unsigned long ftrace_get_parent_ra_addr(unsigned long self_ra, unsigned long |
| 281 | old_parent_ra, unsigned long parent_ra_addr, unsigned long fp) |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 282 | { |
Wu Zhangjin | 2816e32 | 2011-01-20 03:28:30 +0800 | [diff] [blame] | 283 | unsigned long sp, ip, tmp; |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 284 | unsigned int code; |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 285 | int faulted; |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 286 | |
Wu Zhangjin | 68ccf75 | 2010-05-14 19:08:33 +0800 | [diff] [blame] | 287 | /* |
Wu Zhangjin | 2816e32 | 2011-01-20 03:28:30 +0800 | [diff] [blame] | 288 | * For module, move the ip from the return address after the |
Wu Zhangjin | 9a620a5 | 2011-01-20 03:28:27 +0800 | [diff] [blame] | 289 | * instruction "lui v1, hi_16bit_of_mcount"(offset is 24), but for |
| 290 | * kernel, move after the instruction "move ra, at"(offset is 16) |
Wu Zhangjin | 68ccf75 | 2010-05-14 19:08:33 +0800 | [diff] [blame] | 291 | */ |
Wu Zhangjin | 2816e32 | 2011-01-20 03:28:30 +0800 | [diff] [blame] | 292 | ip = self_ra - (in_kernel_space(self_ra) ? 16 : 24); |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 293 | |
Wu Zhangjin | 68ccf75 | 2010-05-14 19:08:33 +0800 | [diff] [blame] | 294 | /* |
| 295 | * search the text until finding the non-store instruction or "s{d,w} |
| 296 | * ra, offset(sp)" instruction |
| 297 | */ |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 298 | do { |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 299 | /* get the code at "ip": code = *(unsigned int *)ip; */ |
| 300 | safe_load_code(code, ip, faulted); |
| 301 | |
| 302 | if (unlikely(faulted)) |
| 303 | return 0; |
Wu Zhangjin | 68ccf75 | 2010-05-14 19:08:33 +0800 | [diff] [blame] | 304 | /* |
| 305 | * If we hit the non-store instruction before finding where the |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 306 | * ra is stored, then this is a leaf function and it does not |
Wu Zhangjin | 68ccf75 | 2010-05-14 19:08:33 +0800 | [diff] [blame] | 307 | * store the ra on the stack |
| 308 | */ |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 309 | if ((code & S_R_SP) != S_R_SP) |
Wu Zhangjin | 2816e32 | 2011-01-20 03:28:30 +0800 | [diff] [blame] | 310 | return parent_ra_addr; |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 311 | |
Wu Zhangjin | 9a620a5 | 2011-01-20 03:28:27 +0800 | [diff] [blame] | 312 | /* Move to the next instruction */ |
| 313 | ip -= 4; |
| 314 | } while ((code & S_RA_SP) != S_RA_SP); |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 315 | |
| 316 | sp = fp + (code & OFFSET_MASK); |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 317 | |
Wu Zhangjin | 2816e32 | 2011-01-20 03:28:30 +0800 | [diff] [blame] | 318 | /* tmp = *(unsigned long *)sp; */ |
| 319 | safe_load_stack(tmp, sp, faulted); |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 320 | if (unlikely(faulted)) |
| 321 | return 0; |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 322 | |
Wu Zhangjin | 2816e32 | 2011-01-20 03:28:30 +0800 | [diff] [blame] | 323 | if (tmp == old_parent_ra) |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 324 | return sp; |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 325 | return 0; |
| 326 | } |
| 327 | |
Wu Zhangjin | 68ccf75 | 2010-05-14 19:08:33 +0800 | [diff] [blame] | 328 | #endif /* !KBUILD_MCOUNT_RA_ADDRESS */ |
Wu Zhangjin | 7326c4e | 2009-11-20 20:34:38 +0800 | [diff] [blame] | 329 | |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 330 | /* |
| 331 | * Hook the return address and push it in the stack of return addrs |
| 332 | * in current thread info. |
| 333 | */ |
Wu Zhangjin | 2816e32 | 2011-01-20 03:28:30 +0800 | [diff] [blame] | 334 | void prepare_ftrace_return(unsigned long *parent_ra_addr, unsigned long self_ra, |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 335 | unsigned long fp) |
| 336 | { |
Wu Zhangjin | 2816e32 | 2011-01-20 03:28:30 +0800 | [diff] [blame] | 337 | unsigned long old_parent_ra; |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 338 | struct ftrace_graph_ent trace; |
| 339 | unsigned long return_hooker = (unsigned long) |
| 340 | &return_to_handler; |
Wu Zhangjin | b9f07eb | 2011-01-22 02:01:53 +0800 | [diff] [blame] | 341 | int faulted, insns; |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 342 | |
Steven Rostedt (Red Hat) | 6a8a505 | 2014-06-25 09:53:21 -0400 | [diff] [blame] | 343 | if (unlikely(ftrace_graph_is_dead())) |
| 344 | return; |
| 345 | |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 346 | if (unlikely(atomic_read(¤t->tracing_graph_pause))) |
| 347 | return; |
| 348 | |
Wu Zhangjin | 68ccf75 | 2010-05-14 19:08:33 +0800 | [diff] [blame] | 349 | /* |
Wu Zhangjin | 2816e32 | 2011-01-20 03:28:30 +0800 | [diff] [blame] | 350 | * "parent_ra_addr" is the stack address saved the return address of |
| 351 | * the caller of _mcount. |
Wu Zhangjin | 7326c4e | 2009-11-20 20:34:38 +0800 | [diff] [blame] | 352 | * |
| 353 | * if the gcc < 4.5, a leaf function does not save the return address |
| 354 | * in the stack address, so, we "emulate" one in _mcount's stack space, |
| 355 | * and hijack it directly, but for a non-leaf function, it save the |
| 356 | * return address to the its own stack space, we can not hijack it |
| 357 | * directly, but need to find the real stack address, |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 358 | * ftrace_get_parent_addr() does it! |
Wu Zhangjin | 7326c4e | 2009-11-20 20:34:38 +0800 | [diff] [blame] | 359 | * |
| 360 | * if gcc>= 4.5, with the new -mmcount-ra-address option, for a |
| 361 | * non-leaf function, the location of the return address will be saved |
| 362 | * to $12 for us, and for a leaf function, only put a zero into $12. we |
| 363 | * do it in ftrace_graph_caller of mcount.S. |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 364 | */ |
| 365 | |
Wu Zhangjin | 2816e32 | 2011-01-20 03:28:30 +0800 | [diff] [blame] | 366 | /* old_parent_ra = *parent_ra_addr; */ |
| 367 | safe_load_stack(old_parent_ra, parent_ra_addr, faulted); |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 368 | if (unlikely(faulted)) |
| 369 | goto out; |
Wu Zhangjin | 7326c4e | 2009-11-20 20:34:38 +0800 | [diff] [blame] | 370 | #ifndef KBUILD_MCOUNT_RA_ADDRESS |
Wu Zhangjin | 2816e32 | 2011-01-20 03:28:30 +0800 | [diff] [blame] | 371 | parent_ra_addr = (unsigned long *)ftrace_get_parent_ra_addr(self_ra, |
| 372 | old_parent_ra, (unsigned long)parent_ra_addr, fp); |
Wu Zhangjin | 68ccf75 | 2010-05-14 19:08:33 +0800 | [diff] [blame] | 373 | /* |
| 374 | * If fails when getting the stack address of the non-leaf function's |
| 375 | * ra, stop function graph tracer and return |
| 376 | */ |
Wu Zhangjin | 2816e32 | 2011-01-20 03:28:30 +0800 | [diff] [blame] | 377 | if (parent_ra_addr == 0) |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 378 | goto out; |
Wu Zhangjin | 7326c4e | 2009-11-20 20:34:38 +0800 | [diff] [blame] | 379 | #endif |
Wu Zhangjin | 2816e32 | 2011-01-20 03:28:30 +0800 | [diff] [blame] | 380 | /* *parent_ra_addr = return_hooker; */ |
| 381 | safe_store_stack(return_hooker, parent_ra_addr, faulted); |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 382 | if (unlikely(faulted)) |
| 383 | goto out; |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 384 | |
Josh Poimboeuf | 9a7c348 | 2016-08-19 06:52:57 -0500 | [diff] [blame^] | 385 | if (ftrace_push_return_trace(old_parent_ra, self_ra, &trace.depth, fp, |
| 386 | NULL) == -EBUSY) { |
Wu Zhangjin | 2816e32 | 2011-01-20 03:28:30 +0800 | [diff] [blame] | 387 | *parent_ra_addr = old_parent_ra; |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 388 | return; |
| 389 | } |
| 390 | |
Wu Zhangjin | b9f07eb | 2011-01-22 02:01:53 +0800 | [diff] [blame] | 391 | /* |
| 392 | * Get the recorded ip of the current mcount calling site in the |
| 393 | * __mcount_loc section, which will be used to filter the function |
| 394 | * entries configured through the tracing/set_graph_function interface. |
| 395 | */ |
| 396 | |
| 397 | insns = in_kernel_space(self_ra) ? 2 : MCOUNT_OFFSET_INSNS + 1; |
| 398 | trace.func = self_ra - (MCOUNT_INSN_SIZE * insns); |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 399 | |
| 400 | /* Only trace if the calling function expects to */ |
| 401 | if (!ftrace_graph_entry(&trace)) { |
| 402 | current->curr_ret_stack--; |
Wu Zhangjin | 2816e32 | 2011-01-20 03:28:30 +0800 | [diff] [blame] | 403 | *parent_ra_addr = old_parent_ra; |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 404 | } |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 405 | return; |
| 406 | out: |
| 407 | ftrace_graph_stop(); |
| 408 | WARN_ON(1); |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 409 | } |
Wu Zhangjin | 68ccf75 | 2010-05-14 19:08:33 +0800 | [diff] [blame] | 410 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
Ralf Baechle | 19e2e17 | 2012-07-13 23:38:17 +0200 | [diff] [blame] | 411 | |
| 412 | #ifdef CONFIG_FTRACE_SYSCALLS |
| 413 | |
| 414 | #ifdef CONFIG_32BIT |
| 415 | unsigned long __init arch_syscall_addr(int nr) |
| 416 | { |
| 417 | return (unsigned long)sys_call_table[nr - __NR_O32_Linux]; |
| 418 | } |
| 419 | #endif |
| 420 | |
| 421 | #ifdef CONFIG_64BIT |
| 422 | |
| 423 | unsigned long __init arch_syscall_addr(int nr) |
| 424 | { |
| 425 | #ifdef CONFIG_MIPS32_N32 |
| 426 | if (nr >= __NR_N32_Linux && nr <= __NR_N32_Linux + __NR_N32_Linux_syscalls) |
Ralf Baechle | 46e12c0 | 2012-07-14 09:22:05 +0200 | [diff] [blame] | 427 | return (unsigned long)sysn32_call_table[nr - __NR_N32_Linux]; |
Ralf Baechle | 19e2e17 | 2012-07-13 23:38:17 +0200 | [diff] [blame] | 428 | #endif |
| 429 | if (nr >= __NR_64_Linux && nr <= __NR_64_Linux + __NR_64_Linux_syscalls) |
| 430 | return (unsigned long)sys_call_table[nr - __NR_64_Linux]; |
| 431 | #ifdef CONFIG_MIPS32_O32 |
| 432 | if (nr >= __NR_O32_Linux && nr <= __NR_O32_Linux + __NR_O32_Linux_syscalls) |
| 433 | return (unsigned long)sys32_call_table[nr - __NR_O32_Linux]; |
| 434 | #endif |
| 435 | |
| 436 | return (unsigned long) &sys_ni_syscall; |
| 437 | } |
| 438 | #endif |
| 439 | |
| 440 | #endif /* CONFIG_FTRACE_SYSCALLS */ |