Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 1 | #!/usr/bin/env perl |
| 2 | |
| 3 | # ==================================================================== |
| 4 | # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL |
| 5 | # project. The module is, however, dual licensed under OpenSSL and |
| 6 | # CRYPTOGAMS licenses depending on where you obtain it. For further |
| 7 | # details see http://www.openssl.org/~appro/cryptogams/. |
| 8 | # ==================================================================== |
| 9 | |
| 10 | # January 2007. |
| 11 | |
| 12 | # Montgomery multiplication for ARMv4. |
| 13 | # |
| 14 | # Performance improvement naturally varies among CPU implementations |
| 15 | # and compilers. The code was observed to provide +65-35% improvement |
| 16 | # [depending on key length, less for longer keys] on ARM920T, and |
| 17 | # +115-80% on Intel IXP425. This is compared to pre-bn_mul_mont code |
| 18 | # base and compiler generated code with in-lined umull and even umlal |
Robert Sloan | a94fe05 | 2017-02-21 08:49:28 -0800 | [diff] [blame] | 19 | # instructions. The latter means that this code didn't really have an |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 20 | # "advantage" of utilizing some "secret" instruction. |
| 21 | # |
| 22 | # The code is interoperable with Thumb ISA and is rather compact, less |
| 23 | # than 1/2KB. Windows CE port would be trivial, as it's exclusively |
| 24 | # about decorations, ABI and instruction syntax are identical. |
| 25 | |
| 26 | # November 2013 |
| 27 | # |
| 28 | # Add NEON code path, which handles lengths divisible by 8. RSA/DSA |
| 29 | # performance improvement on Cortex-A8 is ~45-100% depending on key |
| 30 | # length, more for longer keys. On Cortex-A15 the span is ~10-105%. |
| 31 | # On Snapdragon S4 improvement was measured to vary from ~70% to |
| 32 | # incredible ~380%, yes, 4.8x faster, for RSA4096 sign. But this is |
| 33 | # rather because original integer-only code seems to perform |
| 34 | # suboptimally on S4. Situation on Cortex-A9 is unfortunately |
| 35 | # different. It's being looked into, but the trouble is that |
| 36 | # performance for vectors longer than 256 bits is actually couple |
| 37 | # of percent worse than for integer-only code. The code is chosen |
| 38 | # for execution on all NEON-capable processors, because gain on |
| 39 | # others outweighs the marginal loss on Cortex-A9. |
| 40 | |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 41 | $flavour = shift; |
David Benjamin | c895d6b | 2016-08-11 13:26:41 -0400 | [diff] [blame] | 42 | if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; } |
| 43 | else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} } |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 44 | |
| 45 | if ($flavour && $flavour ne "void") { |
| 46 | $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; |
| 47 | ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or |
| 48 | ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or |
| 49 | die "can't locate arm-xlate.pl"; |
| 50 | |
| 51 | open STDOUT,"| \"$^X\" $xlate $flavour $output"; |
| 52 | } else { |
| 53 | open STDOUT,">$output"; |
| 54 | } |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 55 | |
| 56 | $num="r0"; # starts as num argument, but holds &tp[num-1] |
| 57 | $ap="r1"; |
| 58 | $bp="r2"; $bi="r2"; $rp="r2"; |
| 59 | $np="r3"; |
| 60 | $tp="r4"; |
| 61 | $aj="r5"; |
| 62 | $nj="r6"; |
| 63 | $tj="r7"; |
| 64 | $n0="r8"; |
| 65 | ########### # r9 is reserved by ELF as platform specific, e.g. TLS pointer |
| 66 | $alo="r10"; # sl, gcc uses it to keep @GOT |
| 67 | $ahi="r11"; # fp |
| 68 | $nlo="r12"; # ip |
| 69 | ########### # r13 is stack pointer |
| 70 | $nhi="r14"; # lr |
| 71 | ########### # r15 is program counter |
| 72 | |
| 73 | #### argument block layout relative to &tp[num-1], a.k.a. $num |
| 74 | $_rp="$num,#12*4"; |
| 75 | # ap permanently resides in r1 |
| 76 | $_bp="$num,#13*4"; |
| 77 | # np permanently resides in r3 |
| 78 | $_n0="$num,#14*4"; |
| 79 | $_num="$num,#15*4"; $_bpend=$_num; |
| 80 | |
| 81 | $code=<<___; |
Kenny Root | b849459 | 2015-09-25 02:29:14 +0000 | [diff] [blame] | 82 | #include <openssl/arm_arch.h> |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 83 | |
| 84 | .text |
| 85 | .code 32 |
| 86 | |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 87 | #if __ARM_MAX_ARCH__>=7 |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 88 | .align 5 |
| 89 | .LOPENSSL_armcap: |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 90 | .word OPENSSL_armcap_P-.Lbn_mul_mont |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 91 | #endif |
| 92 | |
| 93 | .global bn_mul_mont |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 94 | .type bn_mul_mont,%function |
| 95 | |
| 96 | .align 5 |
| 97 | bn_mul_mont: |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 98 | .Lbn_mul_mont: |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 99 | ldr ip,[sp,#4] @ load num |
| 100 | stmdb sp!,{r0,r2} @ sp points at argument block |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 101 | #if __ARM_MAX_ARCH__>=7 |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 102 | tst ip,#7 |
| 103 | bne .Lialu |
| 104 | adr r0,bn_mul_mont |
| 105 | ldr r2,.LOPENSSL_armcap |
| 106 | ldr r0,[r0,r2] |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 107 | #ifdef __APPLE__ |
| 108 | ldr r0,[r0] |
| 109 | #endif |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 110 | tst r0,#ARMV7_NEON @ NEON available? |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 111 | ldmia sp, {r0,r2} |
| 112 | beq .Lialu |
| 113 | add sp,sp,#8 |
| 114 | b bn_mul8x_mont_neon |
| 115 | .align 4 |
| 116 | .Lialu: |
| 117 | #endif |
| 118 | cmp ip,#2 |
| 119 | mov $num,ip @ load num |
| 120 | movlt r0,#0 |
| 121 | addlt sp,sp,#2*4 |
| 122 | blt .Labrt |
| 123 | |
| 124 | stmdb sp!,{r4-r12,lr} @ save 10 registers |
| 125 | |
| 126 | mov $num,$num,lsl#2 @ rescale $num for byte count |
| 127 | sub sp,sp,$num @ alloca(4*num) |
| 128 | sub sp,sp,#4 @ +extra dword |
| 129 | sub $num,$num,#4 @ "num=num-1" |
| 130 | add $tp,$bp,$num @ &bp[num-1] |
| 131 | |
| 132 | add $num,sp,$num @ $num to point at &tp[num-1] |
| 133 | ldr $n0,[$_n0] @ &n0 |
| 134 | ldr $bi,[$bp] @ bp[0] |
| 135 | ldr $aj,[$ap],#4 @ ap[0],ap++ |
| 136 | ldr $nj,[$np],#4 @ np[0],np++ |
| 137 | ldr $n0,[$n0] @ *n0 |
| 138 | str $tp,[$_bpend] @ save &bp[num] |
| 139 | |
| 140 | umull $alo,$ahi,$aj,$bi @ ap[0]*bp[0] |
| 141 | str $n0,[$_n0] @ save n0 value |
| 142 | mul $n0,$alo,$n0 @ "tp[0]"*n0 |
| 143 | mov $nlo,#0 |
| 144 | umlal $alo,$nlo,$nj,$n0 @ np[0]*n0+"t[0]" |
| 145 | mov $tp,sp |
| 146 | |
| 147 | .L1st: |
| 148 | ldr $aj,[$ap],#4 @ ap[j],ap++ |
| 149 | mov $alo,$ahi |
| 150 | ldr $nj,[$np],#4 @ np[j],np++ |
| 151 | mov $ahi,#0 |
| 152 | umlal $alo,$ahi,$aj,$bi @ ap[j]*bp[0] |
| 153 | mov $nhi,#0 |
| 154 | umlal $nlo,$nhi,$nj,$n0 @ np[j]*n0 |
| 155 | adds $nlo,$nlo,$alo |
| 156 | str $nlo,[$tp],#4 @ tp[j-1]=,tp++ |
| 157 | adc $nlo,$nhi,#0 |
| 158 | cmp $tp,$num |
| 159 | bne .L1st |
| 160 | |
| 161 | adds $nlo,$nlo,$ahi |
| 162 | ldr $tp,[$_bp] @ restore bp |
| 163 | mov $nhi,#0 |
| 164 | ldr $n0,[$_n0] @ restore n0 |
| 165 | adc $nhi,$nhi,#0 |
| 166 | str $nlo,[$num] @ tp[num-1]= |
| 167 | str $nhi,[$num,#4] @ tp[num]= |
| 168 | |
| 169 | .Louter: |
| 170 | sub $tj,$num,sp @ "original" $num-1 value |
| 171 | sub $ap,$ap,$tj @ "rewind" ap to &ap[1] |
| 172 | ldr $bi,[$tp,#4]! @ *(++bp) |
| 173 | sub $np,$np,$tj @ "rewind" np to &np[1] |
| 174 | ldr $aj,[$ap,#-4] @ ap[0] |
| 175 | ldr $alo,[sp] @ tp[0] |
| 176 | ldr $nj,[$np,#-4] @ np[0] |
| 177 | ldr $tj,[sp,#4] @ tp[1] |
| 178 | |
| 179 | mov $ahi,#0 |
| 180 | umlal $alo,$ahi,$aj,$bi @ ap[0]*bp[i]+tp[0] |
| 181 | str $tp,[$_bp] @ save bp |
| 182 | mul $n0,$alo,$n0 |
| 183 | mov $nlo,#0 |
| 184 | umlal $alo,$nlo,$nj,$n0 @ np[0]*n0+"tp[0]" |
| 185 | mov $tp,sp |
| 186 | |
| 187 | .Linner: |
| 188 | ldr $aj,[$ap],#4 @ ap[j],ap++ |
| 189 | adds $alo,$ahi,$tj @ +=tp[j] |
| 190 | ldr $nj,[$np],#4 @ np[j],np++ |
| 191 | mov $ahi,#0 |
| 192 | umlal $alo,$ahi,$aj,$bi @ ap[j]*bp[i] |
| 193 | mov $nhi,#0 |
| 194 | umlal $nlo,$nhi,$nj,$n0 @ np[j]*n0 |
| 195 | adc $ahi,$ahi,#0 |
| 196 | ldr $tj,[$tp,#8] @ tp[j+1] |
| 197 | adds $nlo,$nlo,$alo |
| 198 | str $nlo,[$tp],#4 @ tp[j-1]=,tp++ |
| 199 | adc $nlo,$nhi,#0 |
| 200 | cmp $tp,$num |
| 201 | bne .Linner |
| 202 | |
| 203 | adds $nlo,$nlo,$ahi |
| 204 | mov $nhi,#0 |
| 205 | ldr $tp,[$_bp] @ restore bp |
| 206 | adc $nhi,$nhi,#0 |
| 207 | ldr $n0,[$_n0] @ restore n0 |
| 208 | adds $nlo,$nlo,$tj |
| 209 | ldr $tj,[$_bpend] @ restore &bp[num] |
| 210 | adc $nhi,$nhi,#0 |
| 211 | str $nlo,[$num] @ tp[num-1]= |
| 212 | str $nhi,[$num,#4] @ tp[num]= |
| 213 | |
| 214 | cmp $tp,$tj |
| 215 | bne .Louter |
| 216 | |
| 217 | ldr $rp,[$_rp] @ pull rp |
| 218 | add $num,$num,#4 @ $num to point at &tp[num] |
| 219 | sub $aj,$num,sp @ "original" num value |
| 220 | mov $tp,sp @ "rewind" $tp |
| 221 | mov $ap,$tp @ "borrow" $ap |
| 222 | sub $np,$np,$aj @ "rewind" $np to &np[0] |
| 223 | |
| 224 | subs $tj,$tj,$tj @ "clear" carry flag |
| 225 | .Lsub: ldr $tj,[$tp],#4 |
| 226 | ldr $nj,[$np],#4 |
| 227 | sbcs $tj,$tj,$nj @ tp[j]-np[j] |
| 228 | str $tj,[$rp],#4 @ rp[j]= |
| 229 | teq $tp,$num @ preserve carry |
| 230 | bne .Lsub |
| 231 | sbcs $nhi,$nhi,#0 @ upmost carry |
| 232 | mov $tp,sp @ "rewind" $tp |
| 233 | sub $rp,$rp,$aj @ "rewind" $rp |
| 234 | |
| 235 | and $ap,$tp,$nhi |
| 236 | bic $np,$rp,$nhi |
| 237 | orr $ap,$ap,$np @ ap=borrow?tp:rp |
| 238 | |
| 239 | .Lcopy: ldr $tj,[$ap],#4 @ copy or in-place refresh |
| 240 | str sp,[$tp],#4 @ zap tp |
| 241 | str $tj,[$rp],#4 |
| 242 | cmp $tp,$num |
| 243 | bne .Lcopy |
| 244 | |
| 245 | add sp,$num,#4 @ skip over tp[num+1] |
| 246 | ldmia sp!,{r4-r12,lr} @ restore registers |
| 247 | add sp,sp,#2*4 @ skip over {r0,r2} |
| 248 | mov r0,#1 |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 249 | .Labrt: |
| 250 | #if __ARM_ARCH__>=5 |
| 251 | ret @ bx lr |
| 252 | #else |
| 253 | tst lr,#1 |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 254 | moveq pc,lr @ be binary compatible with V4, yet |
| 255 | bx lr @ interoperable with Thumb ISA:-) |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 256 | #endif |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 257 | .size bn_mul_mont,.-bn_mul_mont |
| 258 | ___ |
| 259 | { |
| 260 | sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; } |
| 261 | sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; } |
| 262 | |
| 263 | my ($A0,$A1,$A2,$A3)=map("d$_",(0..3)); |
| 264 | my ($N0,$N1,$N2,$N3)=map("d$_",(4..7)); |
| 265 | my ($Z,$Temp)=("q4","q5"); |
| 266 | my ($A0xB,$A1xB,$A2xB,$A3xB,$A4xB,$A5xB,$A6xB,$A7xB)=map("q$_",(6..13)); |
| 267 | my ($Bi,$Ni,$M0)=map("d$_",(28..31)); |
| 268 | my $zero=&Dlo($Z); |
| 269 | my $temp=&Dlo($Temp); |
| 270 | |
| 271 | my ($rptr,$aptr,$bptr,$nptr,$n0,$num)=map("r$_",(0..5)); |
| 272 | my ($tinptr,$toutptr,$inner,$outer)=map("r$_",(6..9)); |
| 273 | |
| 274 | $code.=<<___; |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 275 | #if __ARM_MAX_ARCH__>=7 |
| 276 | .arch armv7-a |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 277 | .fpu neon |
| 278 | |
| 279 | .type bn_mul8x_mont_neon,%function |
| 280 | .align 5 |
| 281 | bn_mul8x_mont_neon: |
| 282 | mov ip,sp |
| 283 | stmdb sp!,{r4-r11} |
| 284 | vstmdb sp!,{d8-d15} @ ABI specification says so |
| 285 | ldmia ip,{r4-r5} @ load rest of parameter block |
| 286 | |
| 287 | sub $toutptr,sp,#16 |
| 288 | vld1.32 {${Bi}[0]}, [$bptr,:32]! |
| 289 | sub $toutptr,$toutptr,$num,lsl#4 |
| 290 | vld1.32 {$A0-$A3}, [$aptr]! @ can't specify :32 :-( |
| 291 | and $toutptr,$toutptr,#-64 |
| 292 | vld1.32 {${M0}[0]}, [$n0,:32] |
| 293 | mov sp,$toutptr @ alloca |
| 294 | veor $zero,$zero,$zero |
| 295 | subs $inner,$num,#8 |
| 296 | vzip.16 $Bi,$zero |
| 297 | |
| 298 | vmull.u32 $A0xB,$Bi,${A0}[0] |
| 299 | vmull.u32 $A1xB,$Bi,${A0}[1] |
| 300 | vmull.u32 $A2xB,$Bi,${A1}[0] |
| 301 | vshl.i64 $temp,`&Dhi("$A0xB")`,#16 |
| 302 | vmull.u32 $A3xB,$Bi,${A1}[1] |
| 303 | |
| 304 | vadd.u64 $temp,$temp,`&Dlo("$A0xB")` |
| 305 | veor $zero,$zero,$zero |
| 306 | vmul.u32 $Ni,$temp,$M0 |
| 307 | |
| 308 | vmull.u32 $A4xB,$Bi,${A2}[0] |
| 309 | vld1.32 {$N0-$N3}, [$nptr]! |
| 310 | vmull.u32 $A5xB,$Bi,${A2}[1] |
| 311 | vmull.u32 $A6xB,$Bi,${A3}[0] |
| 312 | vzip.16 $Ni,$zero |
| 313 | vmull.u32 $A7xB,$Bi,${A3}[1] |
| 314 | |
| 315 | bne .LNEON_1st |
| 316 | |
| 317 | @ special case for num=8, everything is in register bank... |
| 318 | |
| 319 | vmlal.u32 $A0xB,$Ni,${N0}[0] |
| 320 | sub $outer,$num,#1 |
| 321 | vmlal.u32 $A1xB,$Ni,${N0}[1] |
| 322 | vmlal.u32 $A2xB,$Ni,${N1}[0] |
| 323 | vmlal.u32 $A3xB,$Ni,${N1}[1] |
| 324 | |
| 325 | vmlal.u32 $A4xB,$Ni,${N2}[0] |
| 326 | vmov $Temp,$A0xB |
| 327 | vmlal.u32 $A5xB,$Ni,${N2}[1] |
| 328 | vmov $A0xB,$A1xB |
| 329 | vmlal.u32 $A6xB,$Ni,${N3}[0] |
| 330 | vmov $A1xB,$A2xB |
| 331 | vmlal.u32 $A7xB,$Ni,${N3}[1] |
| 332 | vmov $A2xB,$A3xB |
| 333 | vmov $A3xB,$A4xB |
| 334 | vshr.u64 $temp,$temp,#16 |
| 335 | vmov $A4xB,$A5xB |
| 336 | vmov $A5xB,$A6xB |
| 337 | vadd.u64 $temp,$temp,`&Dhi("$Temp")` |
| 338 | vmov $A6xB,$A7xB |
| 339 | veor $A7xB,$A7xB |
| 340 | vshr.u64 $temp,$temp,#16 |
| 341 | |
| 342 | b .LNEON_outer8 |
| 343 | |
| 344 | .align 4 |
| 345 | .LNEON_outer8: |
| 346 | vld1.32 {${Bi}[0]}, [$bptr,:32]! |
| 347 | veor $zero,$zero,$zero |
| 348 | vzip.16 $Bi,$zero |
| 349 | vadd.u64 `&Dlo("$A0xB")`,`&Dlo("$A0xB")`,$temp |
| 350 | |
| 351 | vmlal.u32 $A0xB,$Bi,${A0}[0] |
| 352 | vmlal.u32 $A1xB,$Bi,${A0}[1] |
| 353 | vmlal.u32 $A2xB,$Bi,${A1}[0] |
| 354 | vshl.i64 $temp,`&Dhi("$A0xB")`,#16 |
| 355 | vmlal.u32 $A3xB,$Bi,${A1}[1] |
| 356 | |
| 357 | vadd.u64 $temp,$temp,`&Dlo("$A0xB")` |
| 358 | veor $zero,$zero,$zero |
| 359 | subs $outer,$outer,#1 |
| 360 | vmul.u32 $Ni,$temp,$M0 |
| 361 | |
| 362 | vmlal.u32 $A4xB,$Bi,${A2}[0] |
| 363 | vmlal.u32 $A5xB,$Bi,${A2}[1] |
| 364 | vmlal.u32 $A6xB,$Bi,${A3}[0] |
| 365 | vzip.16 $Ni,$zero |
| 366 | vmlal.u32 $A7xB,$Bi,${A3}[1] |
| 367 | |
| 368 | vmlal.u32 $A0xB,$Ni,${N0}[0] |
| 369 | vmlal.u32 $A1xB,$Ni,${N0}[1] |
| 370 | vmlal.u32 $A2xB,$Ni,${N1}[0] |
| 371 | vmlal.u32 $A3xB,$Ni,${N1}[1] |
| 372 | |
| 373 | vmlal.u32 $A4xB,$Ni,${N2}[0] |
| 374 | vmov $Temp,$A0xB |
| 375 | vmlal.u32 $A5xB,$Ni,${N2}[1] |
| 376 | vmov $A0xB,$A1xB |
| 377 | vmlal.u32 $A6xB,$Ni,${N3}[0] |
| 378 | vmov $A1xB,$A2xB |
| 379 | vmlal.u32 $A7xB,$Ni,${N3}[1] |
| 380 | vmov $A2xB,$A3xB |
| 381 | vmov $A3xB,$A4xB |
| 382 | vshr.u64 $temp,$temp,#16 |
| 383 | vmov $A4xB,$A5xB |
| 384 | vmov $A5xB,$A6xB |
| 385 | vadd.u64 $temp,$temp,`&Dhi("$Temp")` |
| 386 | vmov $A6xB,$A7xB |
| 387 | veor $A7xB,$A7xB |
| 388 | vshr.u64 $temp,$temp,#16 |
| 389 | |
| 390 | bne .LNEON_outer8 |
| 391 | |
| 392 | vadd.u64 `&Dlo("$A0xB")`,`&Dlo("$A0xB")`,$temp |
| 393 | mov $toutptr,sp |
| 394 | vshr.u64 $temp,`&Dlo("$A0xB")`,#16 |
| 395 | mov $inner,$num |
| 396 | vadd.u64 `&Dhi("$A0xB")`,`&Dhi("$A0xB")`,$temp |
| 397 | add $tinptr,sp,#16 |
| 398 | vshr.u64 $temp,`&Dhi("$A0xB")`,#16 |
| 399 | vzip.16 `&Dlo("$A0xB")`,`&Dhi("$A0xB")` |
| 400 | |
| 401 | b .LNEON_tail2 |
| 402 | |
| 403 | .align 4 |
| 404 | .LNEON_1st: |
| 405 | vmlal.u32 $A0xB,$Ni,${N0}[0] |
| 406 | vld1.32 {$A0-$A3}, [$aptr]! |
| 407 | vmlal.u32 $A1xB,$Ni,${N0}[1] |
| 408 | subs $inner,$inner,#8 |
| 409 | vmlal.u32 $A2xB,$Ni,${N1}[0] |
| 410 | vmlal.u32 $A3xB,$Ni,${N1}[1] |
| 411 | |
| 412 | vmlal.u32 $A4xB,$Ni,${N2}[0] |
| 413 | vld1.32 {$N0-$N1}, [$nptr]! |
| 414 | vmlal.u32 $A5xB,$Ni,${N2}[1] |
| 415 | vst1.64 {$A0xB-$A1xB}, [$toutptr,:256]! |
| 416 | vmlal.u32 $A6xB,$Ni,${N3}[0] |
| 417 | vmlal.u32 $A7xB,$Ni,${N3}[1] |
| 418 | vst1.64 {$A2xB-$A3xB}, [$toutptr,:256]! |
| 419 | |
| 420 | vmull.u32 $A0xB,$Bi,${A0}[0] |
| 421 | vld1.32 {$N2-$N3}, [$nptr]! |
| 422 | vmull.u32 $A1xB,$Bi,${A0}[1] |
| 423 | vst1.64 {$A4xB-$A5xB}, [$toutptr,:256]! |
| 424 | vmull.u32 $A2xB,$Bi,${A1}[0] |
| 425 | vmull.u32 $A3xB,$Bi,${A1}[1] |
| 426 | vst1.64 {$A6xB-$A7xB}, [$toutptr,:256]! |
| 427 | |
| 428 | vmull.u32 $A4xB,$Bi,${A2}[0] |
| 429 | vmull.u32 $A5xB,$Bi,${A2}[1] |
| 430 | vmull.u32 $A6xB,$Bi,${A3}[0] |
| 431 | vmull.u32 $A7xB,$Bi,${A3}[1] |
| 432 | |
| 433 | bne .LNEON_1st |
| 434 | |
| 435 | vmlal.u32 $A0xB,$Ni,${N0}[0] |
| 436 | add $tinptr,sp,#16 |
| 437 | vmlal.u32 $A1xB,$Ni,${N0}[1] |
| 438 | sub $aptr,$aptr,$num,lsl#2 @ rewind $aptr |
| 439 | vmlal.u32 $A2xB,$Ni,${N1}[0] |
| 440 | vld1.64 {$Temp}, [sp,:128] |
| 441 | vmlal.u32 $A3xB,$Ni,${N1}[1] |
| 442 | sub $outer,$num,#1 |
| 443 | |
| 444 | vmlal.u32 $A4xB,$Ni,${N2}[0] |
| 445 | vst1.64 {$A0xB-$A1xB}, [$toutptr,:256]! |
| 446 | vmlal.u32 $A5xB,$Ni,${N2}[1] |
| 447 | vshr.u64 $temp,$temp,#16 |
| 448 | vld1.64 {$A0xB}, [$tinptr, :128]! |
| 449 | vmlal.u32 $A6xB,$Ni,${N3}[0] |
| 450 | vst1.64 {$A2xB-$A3xB}, [$toutptr,:256]! |
| 451 | vmlal.u32 $A7xB,$Ni,${N3}[1] |
| 452 | |
| 453 | vst1.64 {$A4xB-$A5xB}, [$toutptr,:256]! |
| 454 | vadd.u64 $temp,$temp,`&Dhi("$Temp")` |
| 455 | veor $Z,$Z,$Z |
| 456 | vst1.64 {$A6xB-$A7xB}, [$toutptr,:256]! |
| 457 | vld1.64 {$A1xB-$A2xB}, [$tinptr, :256]! |
| 458 | vst1.64 {$Z}, [$toutptr,:128] |
| 459 | vshr.u64 $temp,$temp,#16 |
| 460 | |
| 461 | b .LNEON_outer |
| 462 | |
| 463 | .align 4 |
| 464 | .LNEON_outer: |
| 465 | vld1.32 {${Bi}[0]}, [$bptr,:32]! |
| 466 | sub $nptr,$nptr,$num,lsl#2 @ rewind $nptr |
| 467 | vld1.32 {$A0-$A3}, [$aptr]! |
| 468 | veor $zero,$zero,$zero |
| 469 | mov $toutptr,sp |
| 470 | vzip.16 $Bi,$zero |
| 471 | sub $inner,$num,#8 |
| 472 | vadd.u64 `&Dlo("$A0xB")`,`&Dlo("$A0xB")`,$temp |
| 473 | |
| 474 | vmlal.u32 $A0xB,$Bi,${A0}[0] |
| 475 | vld1.64 {$A3xB-$A4xB},[$tinptr,:256]! |
| 476 | vmlal.u32 $A1xB,$Bi,${A0}[1] |
| 477 | vmlal.u32 $A2xB,$Bi,${A1}[0] |
| 478 | vld1.64 {$A5xB-$A6xB},[$tinptr,:256]! |
| 479 | vmlal.u32 $A3xB,$Bi,${A1}[1] |
| 480 | |
| 481 | vshl.i64 $temp,`&Dhi("$A0xB")`,#16 |
| 482 | veor $zero,$zero,$zero |
| 483 | vadd.u64 $temp,$temp,`&Dlo("$A0xB")` |
| 484 | vld1.64 {$A7xB},[$tinptr,:128]! |
| 485 | vmul.u32 $Ni,$temp,$M0 |
| 486 | |
| 487 | vmlal.u32 $A4xB,$Bi,${A2}[0] |
| 488 | vld1.32 {$N0-$N3}, [$nptr]! |
| 489 | vmlal.u32 $A5xB,$Bi,${A2}[1] |
| 490 | vmlal.u32 $A6xB,$Bi,${A3}[0] |
| 491 | vzip.16 $Ni,$zero |
| 492 | vmlal.u32 $A7xB,$Bi,${A3}[1] |
| 493 | |
| 494 | .LNEON_inner: |
| 495 | vmlal.u32 $A0xB,$Ni,${N0}[0] |
| 496 | vld1.32 {$A0-$A3}, [$aptr]! |
| 497 | vmlal.u32 $A1xB,$Ni,${N0}[1] |
| 498 | subs $inner,$inner,#8 |
| 499 | vmlal.u32 $A2xB,$Ni,${N1}[0] |
| 500 | vmlal.u32 $A3xB,$Ni,${N1}[1] |
| 501 | vst1.64 {$A0xB-$A1xB}, [$toutptr,:256]! |
| 502 | |
| 503 | vmlal.u32 $A4xB,$Ni,${N2}[0] |
| 504 | vld1.64 {$A0xB}, [$tinptr, :128]! |
| 505 | vmlal.u32 $A5xB,$Ni,${N2}[1] |
| 506 | vst1.64 {$A2xB-$A3xB}, [$toutptr,:256]! |
| 507 | vmlal.u32 $A6xB,$Ni,${N3}[0] |
| 508 | vld1.64 {$A1xB-$A2xB}, [$tinptr, :256]! |
| 509 | vmlal.u32 $A7xB,$Ni,${N3}[1] |
| 510 | vst1.64 {$A4xB-$A5xB}, [$toutptr,:256]! |
| 511 | |
| 512 | vmlal.u32 $A0xB,$Bi,${A0}[0] |
| 513 | vld1.64 {$A3xB-$A4xB}, [$tinptr, :256]! |
| 514 | vmlal.u32 $A1xB,$Bi,${A0}[1] |
| 515 | vst1.64 {$A6xB-$A7xB}, [$toutptr,:256]! |
| 516 | vmlal.u32 $A2xB,$Bi,${A1}[0] |
| 517 | vld1.64 {$A5xB-$A6xB}, [$tinptr, :256]! |
| 518 | vmlal.u32 $A3xB,$Bi,${A1}[1] |
| 519 | vld1.32 {$N0-$N3}, [$nptr]! |
| 520 | |
| 521 | vmlal.u32 $A4xB,$Bi,${A2}[0] |
| 522 | vld1.64 {$A7xB}, [$tinptr, :128]! |
| 523 | vmlal.u32 $A5xB,$Bi,${A2}[1] |
| 524 | vmlal.u32 $A6xB,$Bi,${A3}[0] |
| 525 | vmlal.u32 $A7xB,$Bi,${A3}[1] |
| 526 | |
| 527 | bne .LNEON_inner |
| 528 | |
| 529 | vmlal.u32 $A0xB,$Ni,${N0}[0] |
| 530 | add $tinptr,sp,#16 |
| 531 | vmlal.u32 $A1xB,$Ni,${N0}[1] |
| 532 | sub $aptr,$aptr,$num,lsl#2 @ rewind $aptr |
| 533 | vmlal.u32 $A2xB,$Ni,${N1}[0] |
| 534 | vld1.64 {$Temp}, [sp,:128] |
| 535 | vmlal.u32 $A3xB,$Ni,${N1}[1] |
| 536 | subs $outer,$outer,#1 |
| 537 | |
| 538 | vmlal.u32 $A4xB,$Ni,${N2}[0] |
| 539 | vst1.64 {$A0xB-$A1xB}, [$toutptr,:256]! |
| 540 | vmlal.u32 $A5xB,$Ni,${N2}[1] |
| 541 | vld1.64 {$A0xB}, [$tinptr, :128]! |
| 542 | vshr.u64 $temp,$temp,#16 |
| 543 | vst1.64 {$A2xB-$A3xB}, [$toutptr,:256]! |
| 544 | vmlal.u32 $A6xB,$Ni,${N3}[0] |
| 545 | vld1.64 {$A1xB-$A2xB}, [$tinptr, :256]! |
| 546 | vmlal.u32 $A7xB,$Ni,${N3}[1] |
| 547 | |
| 548 | vst1.64 {$A4xB-$A5xB}, [$toutptr,:256]! |
| 549 | vadd.u64 $temp,$temp,`&Dhi("$Temp")` |
| 550 | vst1.64 {$A6xB-$A7xB}, [$toutptr,:256]! |
| 551 | vshr.u64 $temp,$temp,#16 |
| 552 | |
| 553 | bne .LNEON_outer |
| 554 | |
| 555 | mov $toutptr,sp |
| 556 | mov $inner,$num |
| 557 | |
| 558 | .LNEON_tail: |
| 559 | vadd.u64 `&Dlo("$A0xB")`,`&Dlo("$A0xB")`,$temp |
| 560 | vld1.64 {$A3xB-$A4xB}, [$tinptr, :256]! |
| 561 | vshr.u64 $temp,`&Dlo("$A0xB")`,#16 |
| 562 | vadd.u64 `&Dhi("$A0xB")`,`&Dhi("$A0xB")`,$temp |
| 563 | vld1.64 {$A5xB-$A6xB}, [$tinptr, :256]! |
| 564 | vshr.u64 $temp,`&Dhi("$A0xB")`,#16 |
| 565 | vld1.64 {$A7xB}, [$tinptr, :128]! |
| 566 | vzip.16 `&Dlo("$A0xB")`,`&Dhi("$A0xB")` |
| 567 | |
| 568 | .LNEON_tail2: |
| 569 | vadd.u64 `&Dlo("$A1xB")`,`&Dlo("$A1xB")`,$temp |
| 570 | vst1.32 {`&Dlo("$A0xB")`[0]}, [$toutptr, :32]! |
| 571 | vshr.u64 $temp,`&Dlo("$A1xB")`,#16 |
| 572 | vadd.u64 `&Dhi("$A1xB")`,`&Dhi("$A1xB")`,$temp |
| 573 | vshr.u64 $temp,`&Dhi("$A1xB")`,#16 |
| 574 | vzip.16 `&Dlo("$A1xB")`,`&Dhi("$A1xB")` |
| 575 | |
| 576 | vadd.u64 `&Dlo("$A2xB")`,`&Dlo("$A2xB")`,$temp |
| 577 | vst1.32 {`&Dlo("$A1xB")`[0]}, [$toutptr, :32]! |
| 578 | vshr.u64 $temp,`&Dlo("$A2xB")`,#16 |
| 579 | vadd.u64 `&Dhi("$A2xB")`,`&Dhi("$A2xB")`,$temp |
| 580 | vshr.u64 $temp,`&Dhi("$A2xB")`,#16 |
| 581 | vzip.16 `&Dlo("$A2xB")`,`&Dhi("$A2xB")` |
| 582 | |
| 583 | vadd.u64 `&Dlo("$A3xB")`,`&Dlo("$A3xB")`,$temp |
| 584 | vst1.32 {`&Dlo("$A2xB")`[0]}, [$toutptr, :32]! |
| 585 | vshr.u64 $temp,`&Dlo("$A3xB")`,#16 |
| 586 | vadd.u64 `&Dhi("$A3xB")`,`&Dhi("$A3xB")`,$temp |
| 587 | vshr.u64 $temp,`&Dhi("$A3xB")`,#16 |
| 588 | vzip.16 `&Dlo("$A3xB")`,`&Dhi("$A3xB")` |
| 589 | |
| 590 | vadd.u64 `&Dlo("$A4xB")`,`&Dlo("$A4xB")`,$temp |
| 591 | vst1.32 {`&Dlo("$A3xB")`[0]}, [$toutptr, :32]! |
| 592 | vshr.u64 $temp,`&Dlo("$A4xB")`,#16 |
| 593 | vadd.u64 `&Dhi("$A4xB")`,`&Dhi("$A4xB")`,$temp |
| 594 | vshr.u64 $temp,`&Dhi("$A4xB")`,#16 |
| 595 | vzip.16 `&Dlo("$A4xB")`,`&Dhi("$A4xB")` |
| 596 | |
| 597 | vadd.u64 `&Dlo("$A5xB")`,`&Dlo("$A5xB")`,$temp |
| 598 | vst1.32 {`&Dlo("$A4xB")`[0]}, [$toutptr, :32]! |
| 599 | vshr.u64 $temp,`&Dlo("$A5xB")`,#16 |
| 600 | vadd.u64 `&Dhi("$A5xB")`,`&Dhi("$A5xB")`,$temp |
| 601 | vshr.u64 $temp,`&Dhi("$A5xB")`,#16 |
| 602 | vzip.16 `&Dlo("$A5xB")`,`&Dhi("$A5xB")` |
| 603 | |
| 604 | vadd.u64 `&Dlo("$A6xB")`,`&Dlo("$A6xB")`,$temp |
| 605 | vst1.32 {`&Dlo("$A5xB")`[0]}, [$toutptr, :32]! |
| 606 | vshr.u64 $temp,`&Dlo("$A6xB")`,#16 |
| 607 | vadd.u64 `&Dhi("$A6xB")`,`&Dhi("$A6xB")`,$temp |
| 608 | vld1.64 {$A0xB}, [$tinptr, :128]! |
| 609 | vshr.u64 $temp,`&Dhi("$A6xB")`,#16 |
| 610 | vzip.16 `&Dlo("$A6xB")`,`&Dhi("$A6xB")` |
| 611 | |
| 612 | vadd.u64 `&Dlo("$A7xB")`,`&Dlo("$A7xB")`,$temp |
| 613 | vst1.32 {`&Dlo("$A6xB")`[0]}, [$toutptr, :32]! |
| 614 | vshr.u64 $temp,`&Dlo("$A7xB")`,#16 |
| 615 | vadd.u64 `&Dhi("$A7xB")`,`&Dhi("$A7xB")`,$temp |
| 616 | vld1.64 {$A1xB-$A2xB}, [$tinptr, :256]! |
| 617 | vshr.u64 $temp,`&Dhi("$A7xB")`,#16 |
| 618 | vzip.16 `&Dlo("$A7xB")`,`&Dhi("$A7xB")` |
| 619 | subs $inner,$inner,#8 |
| 620 | vst1.32 {`&Dlo("$A7xB")`[0]}, [$toutptr, :32]! |
| 621 | |
| 622 | bne .LNEON_tail |
| 623 | |
| 624 | vst1.32 {${temp}[0]}, [$toutptr, :32] @ top-most bit |
| 625 | sub $nptr,$nptr,$num,lsl#2 @ rewind $nptr |
| 626 | subs $aptr,sp,#0 @ clear carry flag |
| 627 | add $bptr,sp,$num,lsl#2 |
| 628 | |
| 629 | .LNEON_sub: |
| 630 | ldmia $aptr!, {r4-r7} |
| 631 | ldmia $nptr!, {r8-r11} |
| 632 | sbcs r8, r4,r8 |
| 633 | sbcs r9, r5,r9 |
| 634 | sbcs r10,r6,r10 |
| 635 | sbcs r11,r7,r11 |
| 636 | teq $aptr,$bptr @ preserves carry |
| 637 | stmia $rptr!, {r8-r11} |
| 638 | bne .LNEON_sub |
| 639 | |
| 640 | ldr r10, [$aptr] @ load top-most bit |
| 641 | veor q0,q0,q0 |
| 642 | sub r11,$bptr,sp @ this is num*4 |
| 643 | veor q1,q1,q1 |
| 644 | mov $aptr,sp |
| 645 | sub $rptr,$rptr,r11 @ rewind $rptr |
| 646 | mov $nptr,$bptr @ second 3/4th of frame |
| 647 | sbcs r10,r10,#0 @ result is carry flag |
| 648 | |
| 649 | .LNEON_copy_n_zap: |
| 650 | ldmia $aptr!, {r4-r7} |
| 651 | ldmia $rptr, {r8-r11} |
| 652 | movcc r8, r4 |
| 653 | vst1.64 {q0-q1}, [$nptr,:256]! @ wipe |
| 654 | movcc r9, r5 |
| 655 | movcc r10,r6 |
| 656 | vst1.64 {q0-q1}, [$nptr,:256]! @ wipe |
| 657 | movcc r11,r7 |
| 658 | ldmia $aptr, {r4-r7} |
| 659 | stmia $rptr!, {r8-r11} |
| 660 | sub $aptr,$aptr,#16 |
| 661 | ldmia $rptr, {r8-r11} |
| 662 | movcc r8, r4 |
| 663 | vst1.64 {q0-q1}, [$aptr,:256]! @ wipe |
| 664 | movcc r9, r5 |
| 665 | movcc r10,r6 |
| 666 | vst1.64 {q0-q1}, [$nptr,:256]! @ wipe |
| 667 | movcc r11,r7 |
| 668 | teq $aptr,$bptr @ preserves carry |
| 669 | stmia $rptr!, {r8-r11} |
| 670 | bne .LNEON_copy_n_zap |
| 671 | |
| 672 | sub sp,ip,#96 |
| 673 | vldmia sp!,{d8-d15} |
| 674 | ldmia sp!,{r4-r11} |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 675 | ret @ bx lr |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 676 | .size bn_mul8x_mont_neon,.-bn_mul8x_mont_neon |
| 677 | #endif |
| 678 | ___ |
| 679 | } |
| 680 | $code.=<<___; |
| 681 | .asciz "Montgomery multiplication for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>" |
| 682 | .align 2 |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 683 | #if __ARM_MAX_ARCH__>=7 |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 684 | .comm OPENSSL_armcap_P,4,4 |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 685 | .hidden OPENSSL_armcap_P |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 686 | #endif |
| 687 | ___ |
| 688 | |
| 689 | $code =~ s/\`([^\`]*)\`/eval $1/gem; |
| 690 | $code =~ s/\bbx\s+lr\b/.word\t0xe12fff1e/gm; # make it possible to compile with -march=armv4 |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 691 | $code =~ s/\bret\b/bx lr/gm; |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 692 | print $code; |
| 693 | close STDOUT; |