Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Userspace implementations of gettimeofday() and friends. |
| 3 | * |
| 4 | * Copyright (C) 2012 ARM Limited |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify |
| 7 | * it under the terms of the GNU General Public License version 2 as |
| 8 | * published by the Free Software Foundation. |
| 9 | * |
| 10 | * This program is distributed in the hope that it will be useful, |
| 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 13 | * GNU General Public License for more details. |
| 14 | * |
| 15 | * You should have received a copy of the GNU General Public License |
| 16 | * along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 17 | * |
| 18 | * Author: Will Deacon <will.deacon@arm.com> |
| 19 | */ |
| 20 | |
| 21 | #include <linux/linkage.h> |
| 22 | #include <asm/asm-offsets.h> |
| 23 | #include <asm/unistd.h> |
| 24 | |
| 25 | #define NSEC_PER_SEC_LO16 0xca00 |
| 26 | #define NSEC_PER_SEC_HI16 0x3b9a |
| 27 | |
| 28 | vdso_data .req x6 |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 29 | seqcnt .req w7 |
| 30 | w_tmp .req w8 |
| 31 | x_tmp .req x8 |
| 32 | |
| 33 | /* |
| 34 | * Conventions for macro arguments: |
| 35 | * - An argument is write-only if its name starts with "res". |
| 36 | * - All other arguments are read-only, unless otherwise specified. |
| 37 | */ |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 38 | |
| 39 | .macro seqcnt_acquire |
| 40 | 9999: ldr seqcnt, [vdso_data, #VDSO_TB_SEQ_COUNT] |
| 41 | tbnz seqcnt, #0, 9999b |
| 42 | dmb ishld |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 43 | .endm |
| 44 | |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 45 | .macro seqcnt_check fail |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 46 | dmb ishld |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 47 | ldr w_tmp, [vdso_data, #VDSO_TB_SEQ_COUNT] |
| 48 | cmp w_tmp, seqcnt |
| 49 | b.ne \fail |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 50 | .endm |
| 51 | |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 52 | .macro syscall_check fail |
| 53 | ldr w_tmp, [vdso_data, #VDSO_USE_SYSCALL] |
| 54 | cbnz w_tmp, \fail |
| 55 | .endm |
| 56 | |
| 57 | .macro get_nsec_per_sec res |
| 58 | mov \res, #NSEC_PER_SEC_LO16 |
| 59 | movk \res, #NSEC_PER_SEC_HI16, lsl #16 |
| 60 | .endm |
| 61 | |
| 62 | /* |
| 63 | * Returns the clock delta, in nanoseconds left-shifted by the clock |
| 64 | * shift. |
| 65 | */ |
| 66 | .macro get_clock_shifted_nsec res, cycle_last, mult |
| 67 | /* Read the virtual counter. */ |
| 68 | isb |
| 69 | mrs x_tmp, cntvct_el0 |
| 70 | /* Calculate cycle delta and convert to ns. */ |
| 71 | sub \res, x_tmp, \cycle_last |
| 72 | /* We can only guarantee 56 bits of precision. */ |
| 73 | movn x_tmp, #0xff00, lsl #48 |
| 74 | and \res, x_tmp, \res |
| 75 | mul \res, \res, \mult |
| 76 | .endm |
| 77 | |
| 78 | /* |
| 79 | * Returns in res_{sec,nsec} the REALTIME timespec, based on the |
| 80 | * "wall time" (xtime) and the clock_mono delta. |
| 81 | */ |
| 82 | .macro get_ts_realtime res_sec, res_nsec, \ |
| 83 | clock_nsec, xtime_sec, xtime_nsec, nsec_to_sec |
| 84 | add \res_nsec, \clock_nsec, \xtime_nsec |
| 85 | udiv x_tmp, \res_nsec, \nsec_to_sec |
| 86 | add \res_sec, \xtime_sec, x_tmp |
| 87 | msub \res_nsec, x_tmp, \nsec_to_sec, \res_nsec |
| 88 | .endm |
| 89 | |
Kevin Brodsky | 49eea43 | 2016-07-12 11:24:00 +0100 | [diff] [blame] | 90 | /* |
| 91 | * Returns in res_{sec,nsec} the timespec based on the clock_raw delta, |
| 92 | * used for CLOCK_MONOTONIC_RAW. |
| 93 | */ |
| 94 | .macro get_ts_clock_raw res_sec, res_nsec, clock_nsec, nsec_to_sec |
| 95 | udiv \res_sec, \clock_nsec, \nsec_to_sec |
| 96 | msub \res_nsec, \res_sec, \nsec_to_sec, \clock_nsec |
| 97 | .endm |
| 98 | |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 99 | /* sec and nsec are modified in place. */ |
| 100 | .macro add_ts sec, nsec, ts_sec, ts_nsec, nsec_to_sec |
| 101 | /* Add timespec. */ |
| 102 | add \sec, \sec, \ts_sec |
| 103 | add \nsec, \nsec, \ts_nsec |
| 104 | |
| 105 | /* Normalise the new timespec. */ |
| 106 | cmp \nsec, \nsec_to_sec |
| 107 | b.lt 9999f |
| 108 | sub \nsec, \nsec, \nsec_to_sec |
| 109 | add \sec, \sec, #1 |
| 110 | 9999: |
| 111 | cmp \nsec, #0 |
| 112 | b.ge 9998f |
| 113 | add \nsec, \nsec, \nsec_to_sec |
| 114 | sub \sec, \sec, #1 |
| 115 | 9998: |
| 116 | .endm |
| 117 | |
| 118 | .macro clock_gettime_return, shift=0 |
| 119 | .if \shift == 1 |
| 120 | lsr x11, x11, x12 |
| 121 | .endif |
| 122 | stp x10, x11, [x1, #TSPEC_TV_SEC] |
| 123 | mov x0, xzr |
| 124 | ret |
| 125 | .endm |
| 126 | |
| 127 | .macro jump_slot jumptable, index, label |
| 128 | .if (. - \jumptable) != 4 * (\index) |
| 129 | .error "Jump slot index mismatch" |
| 130 | .endif |
| 131 | b \label |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 132 | .endm |
| 133 | |
| 134 | .text |
| 135 | |
| 136 | /* int __kernel_gettimeofday(struct timeval *tv, struct timezone *tz); */ |
| 137 | ENTRY(__kernel_gettimeofday) |
| 138 | .cfi_startproc |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 139 | adr vdso_data, _vdso_data |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 140 | /* If tv is NULL, skip to the timezone code. */ |
| 141 | cbz x0, 2f |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 142 | |
| 143 | /* Compute the time of day. */ |
| 144 | 1: seqcnt_acquire |
| 145 | syscall_check fail=4f |
| 146 | ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST] |
Kevin Brodsky | 49eea43 | 2016-07-12 11:24:00 +0100 | [diff] [blame] | 147 | /* w11 = cs_mono_mult, w12 = cs_shift */ |
| 148 | ldp w11, w12, [vdso_data, #VDSO_CS_MONO_MULT] |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 149 | ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC] |
| 150 | seqcnt_check fail=1b |
| 151 | |
| 152 | get_nsec_per_sec res=x9 |
| 153 | lsl x9, x9, x12 |
| 154 | |
| 155 | get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11 |
| 156 | get_ts_realtime res_sec=x10, res_nsec=x11, \ |
| 157 | clock_nsec=x15, xtime_sec=x13, xtime_nsec=x14, nsec_to_sec=x9 |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 158 | |
| 159 | /* Convert ns to us. */ |
Will Deacon | d91fb5c | 2012-11-29 22:19:01 +0000 | [diff] [blame] | 160 | mov x13, #1000 |
Will Deacon | 45a7905 | 2012-11-29 22:33:29 +0000 | [diff] [blame] | 161 | lsl x13, x13, x12 |
Will Deacon | d91fb5c | 2012-11-29 22:19:01 +0000 | [diff] [blame] | 162 | udiv x11, x11, x13 |
| 163 | stp x10, x11, [x0, #TVAL_TV_SEC] |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 164 | 2: |
| 165 | /* If tz is NULL, return 0. */ |
| 166 | cbz x1, 3f |
| 167 | ldp w4, w5, [vdso_data, #VDSO_TZ_MINWEST] |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 168 | stp w4, w5, [x1, #TZ_MINWEST] |
| 169 | 3: |
| 170 | mov x0, xzr |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 171 | ret |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 172 | 4: |
| 173 | /* Syscall fallback. */ |
| 174 | mov x8, #__NR_gettimeofday |
| 175 | svc #0 |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 176 | ret |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 177 | .cfi_endproc |
| 178 | ENDPROC(__kernel_gettimeofday) |
| 179 | |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 180 | #define JUMPSLOT_MAX CLOCK_MONOTONIC_COARSE |
| 181 | |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 182 | /* int __kernel_clock_gettime(clockid_t clock_id, struct timespec *tp); */ |
| 183 | ENTRY(__kernel_clock_gettime) |
| 184 | .cfi_startproc |
Kevin Brodsky | 49eea43 | 2016-07-12 11:24:00 +0100 | [diff] [blame] | 185 | cmp w0, #JUMPSLOT_MAX |
| 186 | b.hi syscall |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 187 | adr vdso_data, _vdso_data |
Kevin Brodsky | 49eea43 | 2016-07-12 11:24:00 +0100 | [diff] [blame] | 188 | adr x_tmp, jumptable |
| 189 | add x_tmp, x_tmp, w0, uxtw #2 |
| 190 | br x_tmp |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 191 | |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 192 | ALIGN |
| 193 | jumptable: |
| 194 | jump_slot jumptable, CLOCK_REALTIME, realtime |
| 195 | jump_slot jumptable, CLOCK_MONOTONIC, monotonic |
Kevin Brodsky | 49eea43 | 2016-07-12 11:24:00 +0100 | [diff] [blame] | 196 | b syscall |
| 197 | b syscall |
| 198 | jump_slot jumptable, CLOCK_MONOTONIC_RAW, monotonic_raw |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 199 | jump_slot jumptable, CLOCK_REALTIME_COARSE, realtime_coarse |
| 200 | jump_slot jumptable, CLOCK_MONOTONIC_COARSE, monotonic_coarse |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 201 | |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 202 | .if (. - jumptable) != 4 * (JUMPSLOT_MAX + 1) |
| 203 | .error "Wrong jumptable size" |
| 204 | .endif |
Nathan Lynch | 069b918 | 2014-02-05 05:53:04 +0000 | [diff] [blame] | 205 | |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 206 | ALIGN |
| 207 | realtime: |
| 208 | seqcnt_acquire |
| 209 | syscall_check fail=syscall |
| 210 | ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST] |
Kevin Brodsky | 49eea43 | 2016-07-12 11:24:00 +0100 | [diff] [blame] | 211 | /* w11 = cs_mono_mult, w12 = cs_shift */ |
| 212 | ldp w11, w12, [vdso_data, #VDSO_CS_MONO_MULT] |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 213 | ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC] |
| 214 | seqcnt_check fail=realtime |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 215 | |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 216 | /* All computations are done with left-shifted nsecs. */ |
| 217 | get_nsec_per_sec res=x9 |
| 218 | lsl x9, x9, x12 |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 219 | |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 220 | get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11 |
| 221 | get_ts_realtime res_sec=x10, res_nsec=x11, \ |
| 222 | clock_nsec=x15, xtime_sec=x13, xtime_nsec=x14, nsec_to_sec=x9 |
| 223 | clock_gettime_return, shift=1 |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 224 | |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 225 | ALIGN |
| 226 | monotonic: |
| 227 | seqcnt_acquire |
| 228 | syscall_check fail=syscall |
| 229 | ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST] |
Kevin Brodsky | 49eea43 | 2016-07-12 11:24:00 +0100 | [diff] [blame] | 230 | /* w11 = cs_mono_mult, w12 = cs_shift */ |
| 231 | ldp w11, w12, [vdso_data, #VDSO_CS_MONO_MULT] |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 232 | ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC] |
| 233 | ldp x3, x4, [vdso_data, #VDSO_WTM_CLK_SEC] |
| 234 | seqcnt_check fail=monotonic |
Nathan Lynch | 069b918 | 2014-02-05 05:53:04 +0000 | [diff] [blame] | 235 | |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 236 | /* All computations are done with left-shifted nsecs. */ |
| 237 | lsl x4, x4, x12 |
| 238 | get_nsec_per_sec res=x9 |
| 239 | lsl x9, x9, x12 |
| 240 | |
| 241 | get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11 |
| 242 | get_ts_realtime res_sec=x10, res_nsec=x11, \ |
| 243 | clock_nsec=x15, xtime_sec=x13, xtime_nsec=x14, nsec_to_sec=x9 |
| 244 | |
| 245 | add_ts sec=x10, nsec=x11, ts_sec=x3, ts_nsec=x4, nsec_to_sec=x9 |
| 246 | clock_gettime_return, shift=1 |
| 247 | |
| 248 | ALIGN |
Kevin Brodsky | 49eea43 | 2016-07-12 11:24:00 +0100 | [diff] [blame] | 249 | monotonic_raw: |
| 250 | seqcnt_acquire |
| 251 | syscall_check fail=syscall |
| 252 | ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST] |
| 253 | /* w11 = cs_raw_mult, w12 = cs_shift */ |
| 254 | ldp w12, w11, [vdso_data, #VDSO_CS_SHIFT] |
| 255 | ldp x13, x14, [vdso_data, #VDSO_RAW_TIME_SEC] |
| 256 | seqcnt_check fail=monotonic_raw |
| 257 | |
| 258 | /* All computations are done with left-shifted nsecs. */ |
Kevin Brodsky | 49eea43 | 2016-07-12 11:24:00 +0100 | [diff] [blame] | 259 | get_nsec_per_sec res=x9 |
| 260 | lsl x9, x9, x12 |
| 261 | |
| 262 | get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11 |
| 263 | get_ts_clock_raw res_sec=x10, res_nsec=x11, \ |
| 264 | clock_nsec=x15, nsec_to_sec=x9 |
| 265 | |
| 266 | add_ts sec=x10, nsec=x11, ts_sec=x13, ts_nsec=x14, nsec_to_sec=x9 |
| 267 | clock_gettime_return, shift=1 |
| 268 | |
| 269 | ALIGN |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 270 | realtime_coarse: |
| 271 | seqcnt_acquire |
Will Deacon | d91fb5c | 2012-11-29 22:19:01 +0000 | [diff] [blame] | 272 | ldp x10, x11, [vdso_data, #VDSO_XTIME_CRS_SEC] |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 273 | seqcnt_check fail=realtime_coarse |
| 274 | clock_gettime_return |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 275 | |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 276 | ALIGN |
| 277 | monotonic_coarse: |
| 278 | seqcnt_acquire |
| 279 | ldp x10, x11, [vdso_data, #VDSO_XTIME_CRS_SEC] |
Will Deacon | d91fb5c | 2012-11-29 22:19:01 +0000 | [diff] [blame] | 280 | ldp x13, x14, [vdso_data, #VDSO_WTM_CLK_SEC] |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 281 | seqcnt_check fail=monotonic_coarse |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 282 | |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 283 | /* Computations are done in (non-shifted) nsecs. */ |
| 284 | get_nsec_per_sec res=x9 |
| 285 | add_ts sec=x10, nsec=x11, ts_sec=x13, ts_nsec=x14, nsec_to_sec=x9 |
| 286 | clock_gettime_return |
Will Deacon | f84a935 | 2012-11-29 22:11:51 +0000 | [diff] [blame] | 287 | |
Kevin Brodsky | b33f491 | 2016-07-12 11:23:59 +0100 | [diff] [blame] | 288 | ALIGN |
| 289 | syscall: /* Syscall fallback. */ |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 290 | mov x8, #__NR_clock_gettime |
| 291 | svc #0 |
| 292 | ret |
| 293 | .cfi_endproc |
| 294 | ENDPROC(__kernel_clock_gettime) |
| 295 | |
| 296 | /* int __kernel_clock_getres(clockid_t clock_id, struct timespec *res); */ |
| 297 | ENTRY(__kernel_clock_getres) |
| 298 | .cfi_startproc |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 299 | cmp w0, #CLOCK_REALTIME |
| 300 | ccmp w0, #CLOCK_MONOTONIC, #0x4, ne |
Kevin Brodsky | 49eea43 | 2016-07-12 11:24:00 +0100 | [diff] [blame] | 301 | ccmp w0, #CLOCK_MONOTONIC_RAW, #0x4, ne |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 302 | b.ne 1f |
| 303 | |
Vincenzo Frascino | 9f641ee | 2019-04-16 17:14:30 +0100 | [diff] [blame] | 304 | adr vdso_data, _vdso_data |
| 305 | ldr w2, [vdso_data, #CLOCK_REALTIME_RES] |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 306 | b 2f |
| 307 | 1: |
| 308 | cmp w0, #CLOCK_REALTIME_COARSE |
| 309 | ccmp w0, #CLOCK_MONOTONIC_COARSE, #0x4, ne |
| 310 | b.ne 4f |
Vincenzo Frascino | 9f641ee | 2019-04-16 17:14:30 +0100 | [diff] [blame] | 311 | ldr x2, 5f |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 312 | 2: |
Nathan Lynch | e1b6b6c | 2015-02-24 17:21:07 -0600 | [diff] [blame] | 313 | cbz w1, 3f |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 314 | stp xzr, x2, [x1] |
| 315 | |
| 316 | 3: /* res == NULL. */ |
| 317 | mov w0, wzr |
| 318 | ret |
| 319 | |
| 320 | 4: /* Syscall fallback. */ |
| 321 | mov x8, #__NR_clock_getres |
| 322 | svc #0 |
| 323 | ret |
| 324 | 5: |
Will Deacon | 9031fef | 2012-03-05 11:49:31 +0000 | [diff] [blame] | 325 | .quad CLOCK_COARSE_RES |
| 326 | .cfi_endproc |
| 327 | ENDPROC(__kernel_clock_getres) |