Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 1 | /* |
| 2 | * aesce-ccm-core.S - AES-CCM transform for ARMv8 with Crypto Extensions |
| 3 | * |
| 4 | * Copyright (C) 2013 - 2014 Linaro Ltd <ard.biesheuvel@linaro.org> |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify |
| 7 | * it under the terms of the GNU General Public License version 2 as |
| 8 | * published by the Free Software Foundation. |
| 9 | */ |
| 10 | |
| 11 | #include <linux/linkage.h> |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 12 | #include <asm/assembler.h> |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 13 | |
| 14 | .text |
| 15 | .arch armv8-a+crypto |
| 16 | |
| 17 | /* |
| 18 | * void ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes, |
| 19 | * u32 *macp, u8 const rk[], u32 rounds); |
| 20 | */ |
| 21 | ENTRY(ce_aes_ccm_auth_data) |
| 22 | ldr w8, [x3] /* leftover from prev round? */ |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 23 | ld1 {v0.16b}, [x0] /* load mac */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 24 | cbz w8, 1f |
| 25 | sub w8, w8, #16 |
| 26 | eor v1.16b, v1.16b, v1.16b |
| 27 | 0: ldrb w7, [x1], #1 /* get 1 byte of input */ |
| 28 | subs w2, w2, #1 |
| 29 | add w8, w8, #1 |
| 30 | ins v1.b[0], w7 |
| 31 | ext v1.16b, v1.16b, v1.16b, #1 /* rotate in the input bytes */ |
| 32 | beq 8f /* out of input? */ |
| 33 | cbnz w8, 0b |
| 34 | eor v0.16b, v0.16b, v1.16b |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 35 | 1: ld1 {v3.16b}, [x4] /* load first round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 36 | prfm pldl1strm, [x1] |
| 37 | cmp w5, #12 /* which key size? */ |
| 38 | add x6, x4, #16 |
| 39 | sub w7, w5, #2 /* modified # of rounds */ |
| 40 | bmi 2f |
| 41 | bne 5f |
| 42 | mov v5.16b, v3.16b |
| 43 | b 4f |
| 44 | 2: mov v4.16b, v3.16b |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 45 | ld1 {v5.16b}, [x6], #16 /* load 2nd round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 46 | 3: aese v0.16b, v4.16b |
| 47 | aesmc v0.16b, v0.16b |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 48 | 4: ld1 {v3.16b}, [x6], #16 /* load next round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 49 | aese v0.16b, v5.16b |
| 50 | aesmc v0.16b, v0.16b |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 51 | 5: ld1 {v4.16b}, [x6], #16 /* load next round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 52 | subs w7, w7, #3 |
| 53 | aese v0.16b, v3.16b |
| 54 | aesmc v0.16b, v0.16b |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 55 | ld1 {v5.16b}, [x6], #16 /* load next round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 56 | bpl 3b |
| 57 | aese v0.16b, v4.16b |
| 58 | subs w2, w2, #16 /* last data? */ |
| 59 | eor v0.16b, v0.16b, v5.16b /* final round */ |
| 60 | bmi 6f |
| 61 | ld1 {v1.16b}, [x1], #16 /* load next input block */ |
| 62 | eor v0.16b, v0.16b, v1.16b /* xor with mac */ |
| 63 | bne 1b |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 64 | 6: st1 {v0.16b}, [x0] /* store mac */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 65 | beq 10f |
| 66 | adds w2, w2, #16 |
| 67 | beq 10f |
| 68 | mov w8, w2 |
| 69 | 7: ldrb w7, [x1], #1 |
| 70 | umov w6, v0.b[0] |
| 71 | eor w6, w6, w7 |
| 72 | strb w6, [x0], #1 |
| 73 | subs w2, w2, #1 |
| 74 | beq 10f |
| 75 | ext v0.16b, v0.16b, v0.16b, #1 /* rotate out the mac bytes */ |
| 76 | b 7b |
Ard Biesheuvel | 9a0eec8 | 2019-01-24 17:33:45 +0100 | [diff] [blame] | 77 | 8: cbz w8, 91f |
| 78 | mov w7, w8 |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 79 | add w8, w8, #16 |
| 80 | 9: ext v1.16b, v1.16b, v1.16b, #1 |
| 81 | adds w7, w7, #1 |
| 82 | bne 9b |
Ard Biesheuvel | 9a0eec8 | 2019-01-24 17:33:45 +0100 | [diff] [blame] | 83 | 91: eor v0.16b, v0.16b, v1.16b |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 84 | st1 {v0.16b}, [x0] |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 85 | 10: str w8, [x3] |
| 86 | ret |
| 87 | ENDPROC(ce_aes_ccm_auth_data) |
| 88 | |
| 89 | /* |
| 90 | * void ce_aes_ccm_final(u8 mac[], u8 const ctr[], u8 const rk[], |
| 91 | * u32 rounds); |
| 92 | */ |
| 93 | ENTRY(ce_aes_ccm_final) |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 94 | ld1 {v3.16b}, [x2], #16 /* load first round key */ |
| 95 | ld1 {v0.16b}, [x0] /* load mac */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 96 | cmp w3, #12 /* which key size? */ |
| 97 | sub w3, w3, #2 /* modified # of rounds */ |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 98 | ld1 {v1.16b}, [x1] /* load 1st ctriv */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 99 | bmi 0f |
| 100 | bne 3f |
| 101 | mov v5.16b, v3.16b |
| 102 | b 2f |
| 103 | 0: mov v4.16b, v3.16b |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 104 | 1: ld1 {v5.16b}, [x2], #16 /* load next round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 105 | aese v0.16b, v4.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 106 | aesmc v0.16b, v0.16b |
Ard Biesheuvel | 4a97abd | 2015-03-17 18:05:13 +0000 | [diff] [blame] | 107 | aese v1.16b, v4.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 108 | aesmc v1.16b, v1.16b |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 109 | 2: ld1 {v3.16b}, [x2], #16 /* load next round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 110 | aese v0.16b, v5.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 111 | aesmc v0.16b, v0.16b |
Ard Biesheuvel | 4a97abd | 2015-03-17 18:05:13 +0000 | [diff] [blame] | 112 | aese v1.16b, v5.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 113 | aesmc v1.16b, v1.16b |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 114 | 3: ld1 {v4.16b}, [x2], #16 /* load next round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 115 | subs w3, w3, #3 |
| 116 | aese v0.16b, v3.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 117 | aesmc v0.16b, v0.16b |
Ard Biesheuvel | 4a97abd | 2015-03-17 18:05:13 +0000 | [diff] [blame] | 118 | aese v1.16b, v3.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 119 | aesmc v1.16b, v1.16b |
| 120 | bpl 1b |
| 121 | aese v0.16b, v4.16b |
| 122 | aese v1.16b, v4.16b |
| 123 | /* final round key cancels out */ |
| 124 | eor v0.16b, v0.16b, v1.16b /* en-/decrypt the mac */ |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 125 | st1 {v0.16b}, [x0] /* store result */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 126 | ret |
| 127 | ENDPROC(ce_aes_ccm_final) |
| 128 | |
| 129 | .macro aes_ccm_do_crypt,enc |
| 130 | ldr x8, [x6, #8] /* load lower ctr */ |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 131 | ld1 {v0.16b}, [x5] /* load mac */ |
| 132 | CPU_LE( rev x8, x8 ) /* keep swabbed ctr in reg */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 133 | 0: /* outer loop */ |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 134 | ld1 {v1.8b}, [x6] /* load upper ctr */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 135 | prfm pldl1strm, [x1] |
| 136 | add x8, x8, #1 |
| 137 | rev x9, x8 |
| 138 | cmp w4, #12 /* which key size? */ |
| 139 | sub w7, w4, #2 /* get modified # of rounds */ |
| 140 | ins v1.d[1], x9 /* no carry in lower ctr */ |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 141 | ld1 {v3.16b}, [x3] /* load first round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 142 | add x10, x3, #16 |
| 143 | bmi 1f |
| 144 | bne 4f |
| 145 | mov v5.16b, v3.16b |
| 146 | b 3f |
| 147 | 1: mov v4.16b, v3.16b |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 148 | ld1 {v5.16b}, [x10], #16 /* load 2nd round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 149 | 2: /* inner loop: 3 rounds, 2x interleaved */ |
| 150 | aese v0.16b, v4.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 151 | aesmc v0.16b, v0.16b |
Ard Biesheuvel | 4a97abd | 2015-03-17 18:05:13 +0000 | [diff] [blame] | 152 | aese v1.16b, v4.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 153 | aesmc v1.16b, v1.16b |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 154 | 3: ld1 {v3.16b}, [x10], #16 /* load next round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 155 | aese v0.16b, v5.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 156 | aesmc v0.16b, v0.16b |
Ard Biesheuvel | 4a97abd | 2015-03-17 18:05:13 +0000 | [diff] [blame] | 157 | aese v1.16b, v5.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 158 | aesmc v1.16b, v1.16b |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 159 | 4: ld1 {v4.16b}, [x10], #16 /* load next round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 160 | subs w7, w7, #3 |
| 161 | aese v0.16b, v3.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 162 | aesmc v0.16b, v0.16b |
Ard Biesheuvel | 4a97abd | 2015-03-17 18:05:13 +0000 | [diff] [blame] | 163 | aese v1.16b, v3.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 164 | aesmc v1.16b, v1.16b |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 165 | ld1 {v5.16b}, [x10], #16 /* load next round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 166 | bpl 2b |
| 167 | aese v0.16b, v4.16b |
| 168 | aese v1.16b, v4.16b |
| 169 | subs w2, w2, #16 |
| 170 | bmi 6f /* partial block? */ |
| 171 | ld1 {v2.16b}, [x1], #16 /* load next input block */ |
| 172 | .if \enc == 1 |
| 173 | eor v2.16b, v2.16b, v5.16b /* final round enc+mac */ |
| 174 | eor v1.16b, v1.16b, v2.16b /* xor with crypted ctr */ |
| 175 | .else |
| 176 | eor v2.16b, v2.16b, v1.16b /* xor with crypted ctr */ |
| 177 | eor v1.16b, v2.16b, v5.16b /* final round enc */ |
| 178 | .endif |
| 179 | eor v0.16b, v0.16b, v2.16b /* xor mac with pt ^ rk[last] */ |
| 180 | st1 {v1.16b}, [x0], #16 /* write output block */ |
| 181 | bne 0b |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 182 | CPU_LE( rev x8, x8 ) |
| 183 | st1 {v0.16b}, [x5] /* store mac */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 184 | str x8, [x6, #8] /* store lsb end of ctr (BE) */ |
| 185 | 5: ret |
| 186 | |
| 187 | 6: eor v0.16b, v0.16b, v5.16b /* final round mac */ |
| 188 | eor v1.16b, v1.16b, v5.16b /* final round enc */ |
Ard Biesheuvel | d018dc9 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 189 | st1 {v0.16b}, [x5] /* store mac */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 190 | add w2, w2, #16 /* process partial tail block */ |
| 191 | 7: ldrb w9, [x1], #1 /* get 1 byte of input */ |
| 192 | umov w6, v1.b[0] /* get top crypted ctr byte */ |
| 193 | umov w7, v0.b[0] /* get top mac byte */ |
| 194 | .if \enc == 1 |
| 195 | eor w7, w7, w9 |
| 196 | eor w9, w9, w6 |
| 197 | .else |
| 198 | eor w9, w9, w6 |
| 199 | eor w7, w7, w9 |
| 200 | .endif |
| 201 | strb w9, [x0], #1 /* store out byte */ |
| 202 | strb w7, [x5], #1 /* store mac byte */ |
| 203 | subs w2, w2, #1 |
| 204 | beq 5b |
| 205 | ext v0.16b, v0.16b, v0.16b, #1 /* shift out mac byte */ |
| 206 | ext v1.16b, v1.16b, v1.16b, #1 /* shift out ctr byte */ |
| 207 | b 7b |
| 208 | .endm |
| 209 | |
| 210 | /* |
| 211 | * void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes, |
| 212 | * u8 const rk[], u32 rounds, u8 mac[], |
| 213 | * u8 ctr[]); |
| 214 | * void ce_aes_ccm_decrypt(u8 out[], u8 const in[], u32 cbytes, |
| 215 | * u8 const rk[], u32 rounds, u8 mac[], |
| 216 | * u8 ctr[]); |
| 217 | */ |
| 218 | ENTRY(ce_aes_ccm_encrypt) |
| 219 | aes_ccm_do_crypt 1 |
| 220 | ENDPROC(ce_aes_ccm_encrypt) |
| 221 | |
| 222 | ENTRY(ce_aes_ccm_decrypt) |
| 223 | aes_ccm_do_crypt 0 |
| 224 | ENDPROC(ce_aes_ccm_decrypt) |