Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 1 | default rel |
| 2 | %define XMMWORD |
| 3 | %define YMMWORD |
| 4 | %define ZMMWORD |
| 5 | section .text code align=64 |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 6 | |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 7 | |
Robert Sloan | 1c9db53 | 2017-03-13 08:03:59 -0700 | [diff] [blame] | 8 | global rsaz_1024_sqr_avx2 |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 9 | |
Robert Sloan | 1c9db53 | 2017-03-13 08:03:59 -0700 | [diff] [blame] | 10 | ALIGN 64 |
| 11 | rsaz_1024_sqr_avx2: |
| 12 | mov QWORD[8+rsp],rdi ;WIN64 prologue |
| 13 | mov QWORD[16+rsp],rsi |
| 14 | mov rax,rsp |
| 15 | $L$SEH_begin_rsaz_1024_sqr_avx2: |
| 16 | mov rdi,rcx |
| 17 | mov rsi,rdx |
| 18 | mov rdx,r8 |
| 19 | mov rcx,r9 |
| 20 | mov r8,QWORD[40+rsp] |
| 21 | |
| 22 | |
| 23 | |
| 24 | lea rax,[rsp] |
| 25 | |
| 26 | push rbx |
| 27 | |
| 28 | push rbp |
| 29 | |
| 30 | push r12 |
| 31 | |
| 32 | push r13 |
| 33 | |
| 34 | push r14 |
| 35 | |
| 36 | push r15 |
| 37 | |
| 38 | vzeroupper |
| 39 | lea rsp,[((-168))+rsp] |
| 40 | vmovaps XMMWORD[(-216)+rax],xmm6 |
| 41 | vmovaps XMMWORD[(-200)+rax],xmm7 |
| 42 | vmovaps XMMWORD[(-184)+rax],xmm8 |
| 43 | vmovaps XMMWORD[(-168)+rax],xmm9 |
| 44 | vmovaps XMMWORD[(-152)+rax],xmm10 |
| 45 | vmovaps XMMWORD[(-136)+rax],xmm11 |
| 46 | vmovaps XMMWORD[(-120)+rax],xmm12 |
| 47 | vmovaps XMMWORD[(-104)+rax],xmm13 |
| 48 | vmovaps XMMWORD[(-88)+rax],xmm14 |
| 49 | vmovaps XMMWORD[(-72)+rax],xmm15 |
| 50 | $L$sqr_1024_body: |
| 51 | mov rbp,rax |
| 52 | |
| 53 | mov r13,rdx |
| 54 | sub rsp,832 |
| 55 | mov r15,r13 |
| 56 | sub rdi,-128 |
| 57 | sub rsi,-128 |
| 58 | sub r13,-128 |
| 59 | |
| 60 | and r15,4095 |
| 61 | add r15,32*10 |
| 62 | shr r15,12 |
| 63 | vpxor ymm9,ymm9,ymm9 |
| 64 | jz NEAR $L$sqr_1024_no_n_copy |
| 65 | |
| 66 | |
| 67 | |
| 68 | |
| 69 | |
| 70 | sub rsp,32*10 |
| 71 | vmovdqu ymm0,YMMWORD[((0-128))+r13] |
| 72 | and rsp,-2048 |
| 73 | vmovdqu ymm1,YMMWORD[((32-128))+r13] |
| 74 | vmovdqu ymm2,YMMWORD[((64-128))+r13] |
| 75 | vmovdqu ymm3,YMMWORD[((96-128))+r13] |
| 76 | vmovdqu ymm4,YMMWORD[((128-128))+r13] |
| 77 | vmovdqu ymm5,YMMWORD[((160-128))+r13] |
| 78 | vmovdqu ymm6,YMMWORD[((192-128))+r13] |
| 79 | vmovdqu ymm7,YMMWORD[((224-128))+r13] |
| 80 | vmovdqu ymm8,YMMWORD[((256-128))+r13] |
| 81 | lea r13,[((832+128))+rsp] |
| 82 | vmovdqu YMMWORD[(0-128)+r13],ymm0 |
| 83 | vmovdqu YMMWORD[(32-128)+r13],ymm1 |
| 84 | vmovdqu YMMWORD[(64-128)+r13],ymm2 |
| 85 | vmovdqu YMMWORD[(96-128)+r13],ymm3 |
| 86 | vmovdqu YMMWORD[(128-128)+r13],ymm4 |
| 87 | vmovdqu YMMWORD[(160-128)+r13],ymm5 |
| 88 | vmovdqu YMMWORD[(192-128)+r13],ymm6 |
| 89 | vmovdqu YMMWORD[(224-128)+r13],ymm7 |
| 90 | vmovdqu YMMWORD[(256-128)+r13],ymm8 |
| 91 | vmovdqu YMMWORD[(288-128)+r13],ymm9 |
| 92 | |
| 93 | $L$sqr_1024_no_n_copy: |
| 94 | and rsp,-1024 |
| 95 | |
| 96 | vmovdqu ymm1,YMMWORD[((32-128))+rsi] |
| 97 | vmovdqu ymm2,YMMWORD[((64-128))+rsi] |
| 98 | vmovdqu ymm3,YMMWORD[((96-128))+rsi] |
| 99 | vmovdqu ymm4,YMMWORD[((128-128))+rsi] |
| 100 | vmovdqu ymm5,YMMWORD[((160-128))+rsi] |
| 101 | vmovdqu ymm6,YMMWORD[((192-128))+rsi] |
| 102 | vmovdqu ymm7,YMMWORD[((224-128))+rsi] |
| 103 | vmovdqu ymm8,YMMWORD[((256-128))+rsi] |
| 104 | |
| 105 | lea rbx,[192+rsp] |
| 106 | vpbroadcastq ymm15,QWORD[$L$and_mask] |
| 107 | jmp NEAR $L$OOP_GRANDE_SQR_1024 |
| 108 | |
| 109 | ALIGN 32 |
| 110 | $L$OOP_GRANDE_SQR_1024: |
| 111 | lea r9,[((576+128))+rsp] |
| 112 | lea r12,[448+rsp] |
| 113 | |
| 114 | |
| 115 | |
| 116 | |
| 117 | vpaddq ymm1,ymm1,ymm1 |
| 118 | vpbroadcastq ymm10,QWORD[((0-128))+rsi] |
| 119 | vpaddq ymm2,ymm2,ymm2 |
| 120 | vmovdqa YMMWORD[(0-128)+r9],ymm1 |
| 121 | vpaddq ymm3,ymm3,ymm3 |
| 122 | vmovdqa YMMWORD[(32-128)+r9],ymm2 |
| 123 | vpaddq ymm4,ymm4,ymm4 |
| 124 | vmovdqa YMMWORD[(64-128)+r9],ymm3 |
| 125 | vpaddq ymm5,ymm5,ymm5 |
| 126 | vmovdqa YMMWORD[(96-128)+r9],ymm4 |
| 127 | vpaddq ymm6,ymm6,ymm6 |
| 128 | vmovdqa YMMWORD[(128-128)+r9],ymm5 |
| 129 | vpaddq ymm7,ymm7,ymm7 |
| 130 | vmovdqa YMMWORD[(160-128)+r9],ymm6 |
| 131 | vpaddq ymm8,ymm8,ymm8 |
| 132 | vmovdqa YMMWORD[(192-128)+r9],ymm7 |
| 133 | vpxor ymm9,ymm9,ymm9 |
| 134 | vmovdqa YMMWORD[(224-128)+r9],ymm8 |
| 135 | |
| 136 | vpmuludq ymm0,ymm10,YMMWORD[((0-128))+rsi] |
| 137 | vpbroadcastq ymm11,QWORD[((32-128))+rsi] |
| 138 | vmovdqu YMMWORD[(288-192)+rbx],ymm9 |
| 139 | vpmuludq ymm1,ymm1,ymm10 |
| 140 | vmovdqu YMMWORD[(320-448)+r12],ymm9 |
| 141 | vpmuludq ymm2,ymm2,ymm10 |
| 142 | vmovdqu YMMWORD[(352-448)+r12],ymm9 |
| 143 | vpmuludq ymm3,ymm3,ymm10 |
| 144 | vmovdqu YMMWORD[(384-448)+r12],ymm9 |
| 145 | vpmuludq ymm4,ymm4,ymm10 |
| 146 | vmovdqu YMMWORD[(416-448)+r12],ymm9 |
| 147 | vpmuludq ymm5,ymm5,ymm10 |
| 148 | vmovdqu YMMWORD[(448-448)+r12],ymm9 |
| 149 | vpmuludq ymm6,ymm6,ymm10 |
| 150 | vmovdqu YMMWORD[(480-448)+r12],ymm9 |
| 151 | vpmuludq ymm7,ymm7,ymm10 |
| 152 | vmovdqu YMMWORD[(512-448)+r12],ymm9 |
| 153 | vpmuludq ymm8,ymm8,ymm10 |
| 154 | vpbroadcastq ymm10,QWORD[((64-128))+rsi] |
| 155 | vmovdqu YMMWORD[(544-448)+r12],ymm9 |
| 156 | |
| 157 | mov r15,rsi |
| 158 | mov r14d,4 |
| 159 | jmp NEAR $L$sqr_entry_1024 |
| 160 | ALIGN 32 |
| 161 | $L$OOP_SQR_1024: |
| 162 | vpbroadcastq ymm11,QWORD[((32-128))+r15] |
| 163 | vpmuludq ymm0,ymm10,YMMWORD[((0-128))+rsi] |
| 164 | vpaddq ymm0,ymm0,YMMWORD[((0-192))+rbx] |
| 165 | vpmuludq ymm1,ymm10,YMMWORD[((0-128))+r9] |
| 166 | vpaddq ymm1,ymm1,YMMWORD[((32-192))+rbx] |
| 167 | vpmuludq ymm2,ymm10,YMMWORD[((32-128))+r9] |
| 168 | vpaddq ymm2,ymm2,YMMWORD[((64-192))+rbx] |
| 169 | vpmuludq ymm3,ymm10,YMMWORD[((64-128))+r9] |
| 170 | vpaddq ymm3,ymm3,YMMWORD[((96-192))+rbx] |
| 171 | vpmuludq ymm4,ymm10,YMMWORD[((96-128))+r9] |
| 172 | vpaddq ymm4,ymm4,YMMWORD[((128-192))+rbx] |
| 173 | vpmuludq ymm5,ymm10,YMMWORD[((128-128))+r9] |
| 174 | vpaddq ymm5,ymm5,YMMWORD[((160-192))+rbx] |
| 175 | vpmuludq ymm6,ymm10,YMMWORD[((160-128))+r9] |
| 176 | vpaddq ymm6,ymm6,YMMWORD[((192-192))+rbx] |
| 177 | vpmuludq ymm7,ymm10,YMMWORD[((192-128))+r9] |
| 178 | vpaddq ymm7,ymm7,YMMWORD[((224-192))+rbx] |
| 179 | vpmuludq ymm8,ymm10,YMMWORD[((224-128))+r9] |
| 180 | vpbroadcastq ymm10,QWORD[((64-128))+r15] |
| 181 | vpaddq ymm8,ymm8,YMMWORD[((256-192))+rbx] |
| 182 | $L$sqr_entry_1024: |
| 183 | vmovdqu YMMWORD[(0-192)+rbx],ymm0 |
| 184 | vmovdqu YMMWORD[(32-192)+rbx],ymm1 |
| 185 | |
| 186 | vpmuludq ymm12,ymm11,YMMWORD[((32-128))+rsi] |
| 187 | vpaddq ymm2,ymm2,ymm12 |
| 188 | vpmuludq ymm14,ymm11,YMMWORD[((32-128))+r9] |
| 189 | vpaddq ymm3,ymm3,ymm14 |
| 190 | vpmuludq ymm13,ymm11,YMMWORD[((64-128))+r9] |
| 191 | vpaddq ymm4,ymm4,ymm13 |
| 192 | vpmuludq ymm12,ymm11,YMMWORD[((96-128))+r9] |
| 193 | vpaddq ymm5,ymm5,ymm12 |
| 194 | vpmuludq ymm14,ymm11,YMMWORD[((128-128))+r9] |
| 195 | vpaddq ymm6,ymm6,ymm14 |
| 196 | vpmuludq ymm13,ymm11,YMMWORD[((160-128))+r9] |
| 197 | vpaddq ymm7,ymm7,ymm13 |
| 198 | vpmuludq ymm12,ymm11,YMMWORD[((192-128))+r9] |
| 199 | vpaddq ymm8,ymm8,ymm12 |
| 200 | vpmuludq ymm0,ymm11,YMMWORD[((224-128))+r9] |
| 201 | vpbroadcastq ymm11,QWORD[((96-128))+r15] |
| 202 | vpaddq ymm0,ymm0,YMMWORD[((288-192))+rbx] |
| 203 | |
| 204 | vmovdqu YMMWORD[(64-192)+rbx],ymm2 |
| 205 | vmovdqu YMMWORD[(96-192)+rbx],ymm3 |
| 206 | |
| 207 | vpmuludq ymm13,ymm10,YMMWORD[((64-128))+rsi] |
| 208 | vpaddq ymm4,ymm4,ymm13 |
| 209 | vpmuludq ymm12,ymm10,YMMWORD[((64-128))+r9] |
| 210 | vpaddq ymm5,ymm5,ymm12 |
| 211 | vpmuludq ymm14,ymm10,YMMWORD[((96-128))+r9] |
| 212 | vpaddq ymm6,ymm6,ymm14 |
| 213 | vpmuludq ymm13,ymm10,YMMWORD[((128-128))+r9] |
| 214 | vpaddq ymm7,ymm7,ymm13 |
| 215 | vpmuludq ymm12,ymm10,YMMWORD[((160-128))+r9] |
| 216 | vpaddq ymm8,ymm8,ymm12 |
| 217 | vpmuludq ymm14,ymm10,YMMWORD[((192-128))+r9] |
| 218 | vpaddq ymm0,ymm0,ymm14 |
| 219 | vpmuludq ymm1,ymm10,YMMWORD[((224-128))+r9] |
| 220 | vpbroadcastq ymm10,QWORD[((128-128))+r15] |
| 221 | vpaddq ymm1,ymm1,YMMWORD[((320-448))+r12] |
| 222 | |
| 223 | vmovdqu YMMWORD[(128-192)+rbx],ymm4 |
| 224 | vmovdqu YMMWORD[(160-192)+rbx],ymm5 |
| 225 | |
| 226 | vpmuludq ymm12,ymm11,YMMWORD[((96-128))+rsi] |
| 227 | vpaddq ymm6,ymm6,ymm12 |
| 228 | vpmuludq ymm14,ymm11,YMMWORD[((96-128))+r9] |
| 229 | vpaddq ymm7,ymm7,ymm14 |
| 230 | vpmuludq ymm13,ymm11,YMMWORD[((128-128))+r9] |
| 231 | vpaddq ymm8,ymm8,ymm13 |
| 232 | vpmuludq ymm12,ymm11,YMMWORD[((160-128))+r9] |
| 233 | vpaddq ymm0,ymm0,ymm12 |
| 234 | vpmuludq ymm14,ymm11,YMMWORD[((192-128))+r9] |
| 235 | vpaddq ymm1,ymm1,ymm14 |
| 236 | vpmuludq ymm2,ymm11,YMMWORD[((224-128))+r9] |
| 237 | vpbroadcastq ymm11,QWORD[((160-128))+r15] |
| 238 | vpaddq ymm2,ymm2,YMMWORD[((352-448))+r12] |
| 239 | |
| 240 | vmovdqu YMMWORD[(192-192)+rbx],ymm6 |
| 241 | vmovdqu YMMWORD[(224-192)+rbx],ymm7 |
| 242 | |
| 243 | vpmuludq ymm12,ymm10,YMMWORD[((128-128))+rsi] |
| 244 | vpaddq ymm8,ymm8,ymm12 |
| 245 | vpmuludq ymm14,ymm10,YMMWORD[((128-128))+r9] |
| 246 | vpaddq ymm0,ymm0,ymm14 |
| 247 | vpmuludq ymm13,ymm10,YMMWORD[((160-128))+r9] |
| 248 | vpaddq ymm1,ymm1,ymm13 |
| 249 | vpmuludq ymm12,ymm10,YMMWORD[((192-128))+r9] |
| 250 | vpaddq ymm2,ymm2,ymm12 |
| 251 | vpmuludq ymm3,ymm10,YMMWORD[((224-128))+r9] |
| 252 | vpbroadcastq ymm10,QWORD[((192-128))+r15] |
| 253 | vpaddq ymm3,ymm3,YMMWORD[((384-448))+r12] |
| 254 | |
| 255 | vmovdqu YMMWORD[(256-192)+rbx],ymm8 |
| 256 | vmovdqu YMMWORD[(288-192)+rbx],ymm0 |
| 257 | lea rbx,[8+rbx] |
| 258 | |
| 259 | vpmuludq ymm13,ymm11,YMMWORD[((160-128))+rsi] |
| 260 | vpaddq ymm1,ymm1,ymm13 |
| 261 | vpmuludq ymm12,ymm11,YMMWORD[((160-128))+r9] |
| 262 | vpaddq ymm2,ymm2,ymm12 |
| 263 | vpmuludq ymm14,ymm11,YMMWORD[((192-128))+r9] |
| 264 | vpaddq ymm3,ymm3,ymm14 |
| 265 | vpmuludq ymm4,ymm11,YMMWORD[((224-128))+r9] |
| 266 | vpbroadcastq ymm11,QWORD[((224-128))+r15] |
| 267 | vpaddq ymm4,ymm4,YMMWORD[((416-448))+r12] |
| 268 | |
| 269 | vmovdqu YMMWORD[(320-448)+r12],ymm1 |
| 270 | vmovdqu YMMWORD[(352-448)+r12],ymm2 |
| 271 | |
| 272 | vpmuludq ymm12,ymm10,YMMWORD[((192-128))+rsi] |
| 273 | vpaddq ymm3,ymm3,ymm12 |
| 274 | vpmuludq ymm14,ymm10,YMMWORD[((192-128))+r9] |
| 275 | vpbroadcastq ymm0,QWORD[((256-128))+r15] |
| 276 | vpaddq ymm4,ymm4,ymm14 |
| 277 | vpmuludq ymm5,ymm10,YMMWORD[((224-128))+r9] |
| 278 | vpbroadcastq ymm10,QWORD[((0+8-128))+r15] |
| 279 | vpaddq ymm5,ymm5,YMMWORD[((448-448))+r12] |
| 280 | |
| 281 | vmovdqu YMMWORD[(384-448)+r12],ymm3 |
| 282 | vmovdqu YMMWORD[(416-448)+r12],ymm4 |
| 283 | lea r15,[8+r15] |
| 284 | |
| 285 | vpmuludq ymm12,ymm11,YMMWORD[((224-128))+rsi] |
| 286 | vpaddq ymm5,ymm5,ymm12 |
| 287 | vpmuludq ymm6,ymm11,YMMWORD[((224-128))+r9] |
| 288 | vpaddq ymm6,ymm6,YMMWORD[((480-448))+r12] |
| 289 | |
| 290 | vpmuludq ymm7,ymm0,YMMWORD[((256-128))+rsi] |
| 291 | vmovdqu YMMWORD[(448-448)+r12],ymm5 |
| 292 | vpaddq ymm7,ymm7,YMMWORD[((512-448))+r12] |
| 293 | vmovdqu YMMWORD[(480-448)+r12],ymm6 |
| 294 | vmovdqu YMMWORD[(512-448)+r12],ymm7 |
| 295 | lea r12,[8+r12] |
| 296 | |
| 297 | dec r14d |
| 298 | jnz NEAR $L$OOP_SQR_1024 |
| 299 | |
| 300 | vmovdqu ymm8,YMMWORD[256+rsp] |
| 301 | vmovdqu ymm1,YMMWORD[288+rsp] |
| 302 | vmovdqu ymm2,YMMWORD[320+rsp] |
| 303 | lea rbx,[192+rsp] |
| 304 | |
| 305 | vpsrlq ymm14,ymm8,29 |
| 306 | vpand ymm8,ymm8,ymm15 |
| 307 | vpsrlq ymm11,ymm1,29 |
| 308 | vpand ymm1,ymm1,ymm15 |
| 309 | |
| 310 | vpermq ymm14,ymm14,0x93 |
| 311 | vpxor ymm9,ymm9,ymm9 |
| 312 | vpermq ymm11,ymm11,0x93 |
| 313 | |
| 314 | vpblendd ymm10,ymm14,ymm9,3 |
| 315 | vpblendd ymm14,ymm11,ymm14,3 |
| 316 | vpaddq ymm8,ymm8,ymm10 |
| 317 | vpblendd ymm11,ymm9,ymm11,3 |
| 318 | vpaddq ymm1,ymm1,ymm14 |
| 319 | vpaddq ymm2,ymm2,ymm11 |
| 320 | vmovdqu YMMWORD[(288-192)+rbx],ymm1 |
| 321 | vmovdqu YMMWORD[(320-192)+rbx],ymm2 |
| 322 | |
| 323 | mov rax,QWORD[rsp] |
| 324 | mov r10,QWORD[8+rsp] |
| 325 | mov r11,QWORD[16+rsp] |
| 326 | mov r12,QWORD[24+rsp] |
| 327 | vmovdqu ymm1,YMMWORD[32+rsp] |
| 328 | vmovdqu ymm2,YMMWORD[((64-192))+rbx] |
| 329 | vmovdqu ymm3,YMMWORD[((96-192))+rbx] |
| 330 | vmovdqu ymm4,YMMWORD[((128-192))+rbx] |
| 331 | vmovdqu ymm5,YMMWORD[((160-192))+rbx] |
| 332 | vmovdqu ymm6,YMMWORD[((192-192))+rbx] |
| 333 | vmovdqu ymm7,YMMWORD[((224-192))+rbx] |
| 334 | |
| 335 | mov r9,rax |
| 336 | imul eax,ecx |
| 337 | and eax,0x1fffffff |
| 338 | vmovd xmm12,eax |
| 339 | |
| 340 | mov rdx,rax |
| 341 | imul rax,QWORD[((-128))+r13] |
| 342 | vpbroadcastq ymm12,xmm12 |
| 343 | add r9,rax |
| 344 | mov rax,rdx |
| 345 | imul rax,QWORD[((8-128))+r13] |
| 346 | shr r9,29 |
| 347 | add r10,rax |
| 348 | mov rax,rdx |
| 349 | imul rax,QWORD[((16-128))+r13] |
| 350 | add r10,r9 |
| 351 | add r11,rax |
| 352 | imul rdx,QWORD[((24-128))+r13] |
| 353 | add r12,rdx |
| 354 | |
| 355 | mov rax,r10 |
| 356 | imul eax,ecx |
| 357 | and eax,0x1fffffff |
| 358 | |
| 359 | mov r14d,9 |
| 360 | jmp NEAR $L$OOP_REDUCE_1024 |
| 361 | |
| 362 | ALIGN 32 |
| 363 | $L$OOP_REDUCE_1024: |
| 364 | vmovd xmm13,eax |
| 365 | vpbroadcastq ymm13,xmm13 |
| 366 | |
| 367 | vpmuludq ymm10,ymm12,YMMWORD[((32-128))+r13] |
| 368 | mov rdx,rax |
| 369 | imul rax,QWORD[((-128))+r13] |
| 370 | vpaddq ymm1,ymm1,ymm10 |
| 371 | add r10,rax |
| 372 | vpmuludq ymm14,ymm12,YMMWORD[((64-128))+r13] |
| 373 | mov rax,rdx |
| 374 | imul rax,QWORD[((8-128))+r13] |
| 375 | vpaddq ymm2,ymm2,ymm14 |
| 376 | vpmuludq ymm11,ymm12,YMMWORD[((96-128))+r13] |
| 377 | DB 0x67 |
| 378 | add r11,rax |
| 379 | DB 0x67 |
| 380 | mov rax,rdx |
| 381 | imul rax,QWORD[((16-128))+r13] |
| 382 | shr r10,29 |
| 383 | vpaddq ymm3,ymm3,ymm11 |
| 384 | vpmuludq ymm10,ymm12,YMMWORD[((128-128))+r13] |
| 385 | add r12,rax |
| 386 | add r11,r10 |
| 387 | vpaddq ymm4,ymm4,ymm10 |
| 388 | vpmuludq ymm14,ymm12,YMMWORD[((160-128))+r13] |
| 389 | mov rax,r11 |
| 390 | imul eax,ecx |
| 391 | vpaddq ymm5,ymm5,ymm14 |
| 392 | vpmuludq ymm11,ymm12,YMMWORD[((192-128))+r13] |
| 393 | and eax,0x1fffffff |
| 394 | vpaddq ymm6,ymm6,ymm11 |
| 395 | vpmuludq ymm10,ymm12,YMMWORD[((224-128))+r13] |
| 396 | vpaddq ymm7,ymm7,ymm10 |
| 397 | vpmuludq ymm14,ymm12,YMMWORD[((256-128))+r13] |
| 398 | vmovd xmm12,eax |
| 399 | |
| 400 | vpaddq ymm8,ymm8,ymm14 |
| 401 | |
| 402 | vpbroadcastq ymm12,xmm12 |
| 403 | |
| 404 | vpmuludq ymm11,ymm13,YMMWORD[((32-8-128))+r13] |
| 405 | vmovdqu ymm14,YMMWORD[((96-8-128))+r13] |
| 406 | mov rdx,rax |
| 407 | imul rax,QWORD[((-128))+r13] |
| 408 | vpaddq ymm1,ymm1,ymm11 |
| 409 | vpmuludq ymm10,ymm13,YMMWORD[((64-8-128))+r13] |
| 410 | vmovdqu ymm11,YMMWORD[((128-8-128))+r13] |
| 411 | add r11,rax |
| 412 | mov rax,rdx |
| 413 | imul rax,QWORD[((8-128))+r13] |
| 414 | vpaddq ymm2,ymm2,ymm10 |
| 415 | add rax,r12 |
| 416 | shr r11,29 |
| 417 | vpmuludq ymm14,ymm14,ymm13 |
| 418 | vmovdqu ymm10,YMMWORD[((160-8-128))+r13] |
| 419 | add rax,r11 |
| 420 | vpaddq ymm3,ymm3,ymm14 |
| 421 | vpmuludq ymm11,ymm11,ymm13 |
| 422 | vmovdqu ymm14,YMMWORD[((192-8-128))+r13] |
| 423 | DB 0x67 |
| 424 | mov r12,rax |
| 425 | imul eax,ecx |
| 426 | vpaddq ymm4,ymm4,ymm11 |
| 427 | vpmuludq ymm10,ymm10,ymm13 |
| 428 | DB 0xc4,0x41,0x7e,0x6f,0x9d,0x58,0x00,0x00,0x00 |
| 429 | and eax,0x1fffffff |
| 430 | vpaddq ymm5,ymm5,ymm10 |
| 431 | vpmuludq ymm14,ymm14,ymm13 |
| 432 | vmovdqu ymm10,YMMWORD[((256-8-128))+r13] |
| 433 | vpaddq ymm6,ymm6,ymm14 |
| 434 | vpmuludq ymm11,ymm11,ymm13 |
| 435 | vmovdqu ymm9,YMMWORD[((288-8-128))+r13] |
| 436 | vmovd xmm0,eax |
| 437 | imul rax,QWORD[((-128))+r13] |
| 438 | vpaddq ymm7,ymm7,ymm11 |
| 439 | vpmuludq ymm10,ymm10,ymm13 |
| 440 | vmovdqu ymm14,YMMWORD[((32-16-128))+r13] |
| 441 | vpbroadcastq ymm0,xmm0 |
| 442 | vpaddq ymm8,ymm8,ymm10 |
| 443 | vpmuludq ymm9,ymm9,ymm13 |
| 444 | vmovdqu ymm11,YMMWORD[((64-16-128))+r13] |
| 445 | add r12,rax |
| 446 | |
| 447 | vmovdqu ymm13,YMMWORD[((32-24-128))+r13] |
| 448 | vpmuludq ymm14,ymm14,ymm12 |
| 449 | vmovdqu ymm10,YMMWORD[((96-16-128))+r13] |
| 450 | vpaddq ymm1,ymm1,ymm14 |
| 451 | vpmuludq ymm13,ymm13,ymm0 |
| 452 | vpmuludq ymm11,ymm11,ymm12 |
| 453 | DB 0xc4,0x41,0x7e,0x6f,0xb5,0xf0,0xff,0xff,0xff |
| 454 | vpaddq ymm13,ymm13,ymm1 |
| 455 | vpaddq ymm2,ymm2,ymm11 |
| 456 | vpmuludq ymm10,ymm10,ymm12 |
| 457 | vmovdqu ymm11,YMMWORD[((160-16-128))+r13] |
| 458 | DB 0x67 |
| 459 | vmovq rax,xmm13 |
| 460 | vmovdqu YMMWORD[rsp],ymm13 |
| 461 | vpaddq ymm3,ymm3,ymm10 |
| 462 | vpmuludq ymm14,ymm14,ymm12 |
| 463 | vmovdqu ymm10,YMMWORD[((192-16-128))+r13] |
| 464 | vpaddq ymm4,ymm4,ymm14 |
| 465 | vpmuludq ymm11,ymm11,ymm12 |
| 466 | vmovdqu ymm14,YMMWORD[((224-16-128))+r13] |
| 467 | vpaddq ymm5,ymm5,ymm11 |
| 468 | vpmuludq ymm10,ymm10,ymm12 |
| 469 | vmovdqu ymm11,YMMWORD[((256-16-128))+r13] |
| 470 | vpaddq ymm6,ymm6,ymm10 |
| 471 | vpmuludq ymm14,ymm14,ymm12 |
| 472 | shr r12,29 |
| 473 | vmovdqu ymm10,YMMWORD[((288-16-128))+r13] |
| 474 | add rax,r12 |
| 475 | vpaddq ymm7,ymm7,ymm14 |
| 476 | vpmuludq ymm11,ymm11,ymm12 |
| 477 | |
| 478 | mov r9,rax |
| 479 | imul eax,ecx |
| 480 | vpaddq ymm8,ymm8,ymm11 |
| 481 | vpmuludq ymm10,ymm10,ymm12 |
| 482 | and eax,0x1fffffff |
| 483 | vmovd xmm12,eax |
| 484 | vmovdqu ymm11,YMMWORD[((96-24-128))+r13] |
| 485 | DB 0x67 |
| 486 | vpaddq ymm9,ymm9,ymm10 |
| 487 | vpbroadcastq ymm12,xmm12 |
| 488 | |
| 489 | vpmuludq ymm14,ymm0,YMMWORD[((64-24-128))+r13] |
| 490 | vmovdqu ymm10,YMMWORD[((128-24-128))+r13] |
| 491 | mov rdx,rax |
| 492 | imul rax,QWORD[((-128))+r13] |
| 493 | mov r10,QWORD[8+rsp] |
| 494 | vpaddq ymm1,ymm2,ymm14 |
| 495 | vpmuludq ymm11,ymm11,ymm0 |
| 496 | vmovdqu ymm14,YMMWORD[((160-24-128))+r13] |
| 497 | add r9,rax |
| 498 | mov rax,rdx |
| 499 | imul rax,QWORD[((8-128))+r13] |
| 500 | DB 0x67 |
| 501 | shr r9,29 |
| 502 | mov r11,QWORD[16+rsp] |
| 503 | vpaddq ymm2,ymm3,ymm11 |
| 504 | vpmuludq ymm10,ymm10,ymm0 |
| 505 | vmovdqu ymm11,YMMWORD[((192-24-128))+r13] |
| 506 | add r10,rax |
| 507 | mov rax,rdx |
| 508 | imul rax,QWORD[((16-128))+r13] |
| 509 | vpaddq ymm3,ymm4,ymm10 |
| 510 | vpmuludq ymm14,ymm14,ymm0 |
| 511 | vmovdqu ymm10,YMMWORD[((224-24-128))+r13] |
| 512 | imul rdx,QWORD[((24-128))+r13] |
| 513 | add r11,rax |
| 514 | lea rax,[r10*1+r9] |
| 515 | vpaddq ymm4,ymm5,ymm14 |
| 516 | vpmuludq ymm11,ymm11,ymm0 |
| 517 | vmovdqu ymm14,YMMWORD[((256-24-128))+r13] |
| 518 | mov r10,rax |
| 519 | imul eax,ecx |
| 520 | vpmuludq ymm10,ymm10,ymm0 |
| 521 | vpaddq ymm5,ymm6,ymm11 |
| 522 | vmovdqu ymm11,YMMWORD[((288-24-128))+r13] |
| 523 | and eax,0x1fffffff |
| 524 | vpaddq ymm6,ymm7,ymm10 |
| 525 | vpmuludq ymm14,ymm14,ymm0 |
| 526 | add rdx,QWORD[24+rsp] |
| 527 | vpaddq ymm7,ymm8,ymm14 |
| 528 | vpmuludq ymm11,ymm11,ymm0 |
| 529 | vpaddq ymm8,ymm9,ymm11 |
| 530 | vmovq xmm9,r12 |
| 531 | mov r12,rdx |
| 532 | |
| 533 | dec r14d |
| 534 | jnz NEAR $L$OOP_REDUCE_1024 |
| 535 | lea r12,[448+rsp] |
| 536 | vpaddq ymm0,ymm13,ymm9 |
| 537 | vpxor ymm9,ymm9,ymm9 |
| 538 | |
| 539 | vpaddq ymm0,ymm0,YMMWORD[((288-192))+rbx] |
| 540 | vpaddq ymm1,ymm1,YMMWORD[((320-448))+r12] |
| 541 | vpaddq ymm2,ymm2,YMMWORD[((352-448))+r12] |
| 542 | vpaddq ymm3,ymm3,YMMWORD[((384-448))+r12] |
| 543 | vpaddq ymm4,ymm4,YMMWORD[((416-448))+r12] |
| 544 | vpaddq ymm5,ymm5,YMMWORD[((448-448))+r12] |
| 545 | vpaddq ymm6,ymm6,YMMWORD[((480-448))+r12] |
| 546 | vpaddq ymm7,ymm7,YMMWORD[((512-448))+r12] |
| 547 | vpaddq ymm8,ymm8,YMMWORD[((544-448))+r12] |
| 548 | |
| 549 | vpsrlq ymm14,ymm0,29 |
| 550 | vpand ymm0,ymm0,ymm15 |
| 551 | vpsrlq ymm11,ymm1,29 |
| 552 | vpand ymm1,ymm1,ymm15 |
| 553 | vpsrlq ymm12,ymm2,29 |
| 554 | vpermq ymm14,ymm14,0x93 |
| 555 | vpand ymm2,ymm2,ymm15 |
| 556 | vpsrlq ymm13,ymm3,29 |
| 557 | vpermq ymm11,ymm11,0x93 |
| 558 | vpand ymm3,ymm3,ymm15 |
| 559 | vpermq ymm12,ymm12,0x93 |
| 560 | |
| 561 | vpblendd ymm10,ymm14,ymm9,3 |
| 562 | vpermq ymm13,ymm13,0x93 |
| 563 | vpblendd ymm14,ymm11,ymm14,3 |
| 564 | vpaddq ymm0,ymm0,ymm10 |
| 565 | vpblendd ymm11,ymm12,ymm11,3 |
| 566 | vpaddq ymm1,ymm1,ymm14 |
| 567 | vpblendd ymm12,ymm13,ymm12,3 |
| 568 | vpaddq ymm2,ymm2,ymm11 |
| 569 | vpblendd ymm13,ymm9,ymm13,3 |
| 570 | vpaddq ymm3,ymm3,ymm12 |
| 571 | vpaddq ymm4,ymm4,ymm13 |
| 572 | |
| 573 | vpsrlq ymm14,ymm0,29 |
| 574 | vpand ymm0,ymm0,ymm15 |
| 575 | vpsrlq ymm11,ymm1,29 |
| 576 | vpand ymm1,ymm1,ymm15 |
| 577 | vpsrlq ymm12,ymm2,29 |
| 578 | vpermq ymm14,ymm14,0x93 |
| 579 | vpand ymm2,ymm2,ymm15 |
| 580 | vpsrlq ymm13,ymm3,29 |
| 581 | vpermq ymm11,ymm11,0x93 |
| 582 | vpand ymm3,ymm3,ymm15 |
| 583 | vpermq ymm12,ymm12,0x93 |
| 584 | |
| 585 | vpblendd ymm10,ymm14,ymm9,3 |
| 586 | vpermq ymm13,ymm13,0x93 |
| 587 | vpblendd ymm14,ymm11,ymm14,3 |
| 588 | vpaddq ymm0,ymm0,ymm10 |
| 589 | vpblendd ymm11,ymm12,ymm11,3 |
| 590 | vpaddq ymm1,ymm1,ymm14 |
| 591 | vmovdqu YMMWORD[(0-128)+rdi],ymm0 |
| 592 | vpblendd ymm12,ymm13,ymm12,3 |
| 593 | vpaddq ymm2,ymm2,ymm11 |
| 594 | vmovdqu YMMWORD[(32-128)+rdi],ymm1 |
| 595 | vpblendd ymm13,ymm9,ymm13,3 |
| 596 | vpaddq ymm3,ymm3,ymm12 |
| 597 | vmovdqu YMMWORD[(64-128)+rdi],ymm2 |
| 598 | vpaddq ymm4,ymm4,ymm13 |
| 599 | vmovdqu YMMWORD[(96-128)+rdi],ymm3 |
| 600 | vpsrlq ymm14,ymm4,29 |
| 601 | vpand ymm4,ymm4,ymm15 |
| 602 | vpsrlq ymm11,ymm5,29 |
| 603 | vpand ymm5,ymm5,ymm15 |
| 604 | vpsrlq ymm12,ymm6,29 |
| 605 | vpermq ymm14,ymm14,0x93 |
| 606 | vpand ymm6,ymm6,ymm15 |
| 607 | vpsrlq ymm13,ymm7,29 |
| 608 | vpermq ymm11,ymm11,0x93 |
| 609 | vpand ymm7,ymm7,ymm15 |
| 610 | vpsrlq ymm0,ymm8,29 |
| 611 | vpermq ymm12,ymm12,0x93 |
| 612 | vpand ymm8,ymm8,ymm15 |
| 613 | vpermq ymm13,ymm13,0x93 |
| 614 | |
| 615 | vpblendd ymm10,ymm14,ymm9,3 |
| 616 | vpermq ymm0,ymm0,0x93 |
| 617 | vpblendd ymm14,ymm11,ymm14,3 |
| 618 | vpaddq ymm4,ymm4,ymm10 |
| 619 | vpblendd ymm11,ymm12,ymm11,3 |
| 620 | vpaddq ymm5,ymm5,ymm14 |
| 621 | vpblendd ymm12,ymm13,ymm12,3 |
| 622 | vpaddq ymm6,ymm6,ymm11 |
| 623 | vpblendd ymm13,ymm0,ymm13,3 |
| 624 | vpaddq ymm7,ymm7,ymm12 |
| 625 | vpaddq ymm8,ymm8,ymm13 |
| 626 | |
| 627 | vpsrlq ymm14,ymm4,29 |
| 628 | vpand ymm4,ymm4,ymm15 |
| 629 | vpsrlq ymm11,ymm5,29 |
| 630 | vpand ymm5,ymm5,ymm15 |
| 631 | vpsrlq ymm12,ymm6,29 |
| 632 | vpermq ymm14,ymm14,0x93 |
| 633 | vpand ymm6,ymm6,ymm15 |
| 634 | vpsrlq ymm13,ymm7,29 |
| 635 | vpermq ymm11,ymm11,0x93 |
| 636 | vpand ymm7,ymm7,ymm15 |
| 637 | vpsrlq ymm0,ymm8,29 |
| 638 | vpermq ymm12,ymm12,0x93 |
| 639 | vpand ymm8,ymm8,ymm15 |
| 640 | vpermq ymm13,ymm13,0x93 |
| 641 | |
| 642 | vpblendd ymm10,ymm14,ymm9,3 |
| 643 | vpermq ymm0,ymm0,0x93 |
| 644 | vpblendd ymm14,ymm11,ymm14,3 |
| 645 | vpaddq ymm4,ymm4,ymm10 |
| 646 | vpblendd ymm11,ymm12,ymm11,3 |
| 647 | vpaddq ymm5,ymm5,ymm14 |
| 648 | vmovdqu YMMWORD[(128-128)+rdi],ymm4 |
| 649 | vpblendd ymm12,ymm13,ymm12,3 |
| 650 | vpaddq ymm6,ymm6,ymm11 |
| 651 | vmovdqu YMMWORD[(160-128)+rdi],ymm5 |
| 652 | vpblendd ymm13,ymm0,ymm13,3 |
| 653 | vpaddq ymm7,ymm7,ymm12 |
| 654 | vmovdqu YMMWORD[(192-128)+rdi],ymm6 |
| 655 | vpaddq ymm8,ymm8,ymm13 |
| 656 | vmovdqu YMMWORD[(224-128)+rdi],ymm7 |
| 657 | vmovdqu YMMWORD[(256-128)+rdi],ymm8 |
| 658 | |
| 659 | mov rsi,rdi |
| 660 | dec r8d |
| 661 | jne NEAR $L$OOP_GRANDE_SQR_1024 |
| 662 | |
| 663 | vzeroall |
| 664 | mov rax,rbp |
| 665 | |
| 666 | $L$sqr_1024_in_tail: |
| 667 | movaps xmm6,XMMWORD[((-216))+rax] |
| 668 | movaps xmm7,XMMWORD[((-200))+rax] |
| 669 | movaps xmm8,XMMWORD[((-184))+rax] |
| 670 | movaps xmm9,XMMWORD[((-168))+rax] |
| 671 | movaps xmm10,XMMWORD[((-152))+rax] |
| 672 | movaps xmm11,XMMWORD[((-136))+rax] |
| 673 | movaps xmm12,XMMWORD[((-120))+rax] |
| 674 | movaps xmm13,XMMWORD[((-104))+rax] |
| 675 | movaps xmm14,XMMWORD[((-88))+rax] |
| 676 | movaps xmm15,XMMWORD[((-72))+rax] |
| 677 | mov r15,QWORD[((-48))+rax] |
| 678 | |
| 679 | mov r14,QWORD[((-40))+rax] |
| 680 | |
| 681 | mov r13,QWORD[((-32))+rax] |
| 682 | |
| 683 | mov r12,QWORD[((-24))+rax] |
| 684 | |
| 685 | mov rbp,QWORD[((-16))+rax] |
| 686 | |
| 687 | mov rbx,QWORD[((-8))+rax] |
| 688 | |
| 689 | lea rsp,[rax] |
| 690 | |
| 691 | $L$sqr_1024_epilogue: |
| 692 | mov rdi,QWORD[8+rsp] ;WIN64 epilogue |
| 693 | mov rsi,QWORD[16+rsp] |
| 694 | DB 0F3h,0C3h ;repret |
| 695 | |
| 696 | $L$SEH_end_rsaz_1024_sqr_avx2: |
| 697 | global rsaz_1024_mul_avx2 |
| 698 | |
| 699 | ALIGN 64 |
| 700 | rsaz_1024_mul_avx2: |
| 701 | mov QWORD[8+rsp],rdi ;WIN64 prologue |
| 702 | mov QWORD[16+rsp],rsi |
| 703 | mov rax,rsp |
| 704 | $L$SEH_begin_rsaz_1024_mul_avx2: |
| 705 | mov rdi,rcx |
| 706 | mov rsi,rdx |
| 707 | mov rdx,r8 |
| 708 | mov rcx,r9 |
| 709 | mov r8,QWORD[40+rsp] |
| 710 | |
| 711 | |
| 712 | |
| 713 | lea rax,[rsp] |
| 714 | |
| 715 | push rbx |
| 716 | |
| 717 | push rbp |
| 718 | |
| 719 | push r12 |
| 720 | |
| 721 | push r13 |
| 722 | |
| 723 | push r14 |
| 724 | |
| 725 | push r15 |
| 726 | |
| 727 | vzeroupper |
| 728 | lea rsp,[((-168))+rsp] |
| 729 | vmovaps XMMWORD[(-216)+rax],xmm6 |
| 730 | vmovaps XMMWORD[(-200)+rax],xmm7 |
| 731 | vmovaps XMMWORD[(-184)+rax],xmm8 |
| 732 | vmovaps XMMWORD[(-168)+rax],xmm9 |
| 733 | vmovaps XMMWORD[(-152)+rax],xmm10 |
| 734 | vmovaps XMMWORD[(-136)+rax],xmm11 |
| 735 | vmovaps XMMWORD[(-120)+rax],xmm12 |
| 736 | vmovaps XMMWORD[(-104)+rax],xmm13 |
| 737 | vmovaps XMMWORD[(-88)+rax],xmm14 |
| 738 | vmovaps XMMWORD[(-72)+rax],xmm15 |
| 739 | $L$mul_1024_body: |
| 740 | mov rbp,rax |
| 741 | |
| 742 | vzeroall |
| 743 | mov r13,rdx |
| 744 | sub rsp,64 |
| 745 | |
| 746 | |
| 747 | |
| 748 | |
| 749 | |
| 750 | |
| 751 | DB 0x67,0x67 |
| 752 | mov r15,rsi |
| 753 | and r15,4095 |
| 754 | add r15,32*10 |
| 755 | shr r15,12 |
| 756 | mov r15,rsi |
| 757 | cmovnz rsi,r13 |
| 758 | cmovnz r13,r15 |
| 759 | |
| 760 | mov r15,rcx |
| 761 | sub rsi,-128 |
| 762 | sub rcx,-128 |
| 763 | sub rdi,-128 |
| 764 | |
| 765 | and r15,4095 |
| 766 | add r15,32*10 |
| 767 | DB 0x67,0x67 |
| 768 | shr r15,12 |
| 769 | jz NEAR $L$mul_1024_no_n_copy |
| 770 | |
| 771 | |
| 772 | |
| 773 | |
| 774 | |
| 775 | sub rsp,32*10 |
| 776 | vmovdqu ymm0,YMMWORD[((0-128))+rcx] |
| 777 | and rsp,-512 |
| 778 | vmovdqu ymm1,YMMWORD[((32-128))+rcx] |
| 779 | vmovdqu ymm2,YMMWORD[((64-128))+rcx] |
| 780 | vmovdqu ymm3,YMMWORD[((96-128))+rcx] |
| 781 | vmovdqu ymm4,YMMWORD[((128-128))+rcx] |
| 782 | vmovdqu ymm5,YMMWORD[((160-128))+rcx] |
| 783 | vmovdqu ymm6,YMMWORD[((192-128))+rcx] |
| 784 | vmovdqu ymm7,YMMWORD[((224-128))+rcx] |
| 785 | vmovdqu ymm8,YMMWORD[((256-128))+rcx] |
| 786 | lea rcx,[((64+128))+rsp] |
| 787 | vmovdqu YMMWORD[(0-128)+rcx],ymm0 |
| 788 | vpxor ymm0,ymm0,ymm0 |
| 789 | vmovdqu YMMWORD[(32-128)+rcx],ymm1 |
| 790 | vpxor ymm1,ymm1,ymm1 |
| 791 | vmovdqu YMMWORD[(64-128)+rcx],ymm2 |
| 792 | vpxor ymm2,ymm2,ymm2 |
| 793 | vmovdqu YMMWORD[(96-128)+rcx],ymm3 |
| 794 | vpxor ymm3,ymm3,ymm3 |
| 795 | vmovdqu YMMWORD[(128-128)+rcx],ymm4 |
| 796 | vpxor ymm4,ymm4,ymm4 |
| 797 | vmovdqu YMMWORD[(160-128)+rcx],ymm5 |
| 798 | vpxor ymm5,ymm5,ymm5 |
| 799 | vmovdqu YMMWORD[(192-128)+rcx],ymm6 |
| 800 | vpxor ymm6,ymm6,ymm6 |
| 801 | vmovdqu YMMWORD[(224-128)+rcx],ymm7 |
| 802 | vpxor ymm7,ymm7,ymm7 |
| 803 | vmovdqu YMMWORD[(256-128)+rcx],ymm8 |
| 804 | vmovdqa ymm8,ymm0 |
| 805 | vmovdqu YMMWORD[(288-128)+rcx],ymm9 |
| 806 | $L$mul_1024_no_n_copy: |
| 807 | and rsp,-64 |
| 808 | |
| 809 | mov rbx,QWORD[r13] |
| 810 | vpbroadcastq ymm10,QWORD[r13] |
| 811 | vmovdqu YMMWORD[rsp],ymm0 |
| 812 | xor r9,r9 |
| 813 | DB 0x67 |
| 814 | xor r10,r10 |
| 815 | xor r11,r11 |
| 816 | xor r12,r12 |
| 817 | |
| 818 | vmovdqu ymm15,YMMWORD[$L$and_mask] |
| 819 | mov r14d,9 |
| 820 | vmovdqu YMMWORD[(288-128)+rdi],ymm9 |
| 821 | jmp NEAR $L$oop_mul_1024 |
| 822 | |
| 823 | ALIGN 32 |
| 824 | $L$oop_mul_1024: |
| 825 | vpsrlq ymm9,ymm3,29 |
| 826 | mov rax,rbx |
| 827 | imul rax,QWORD[((-128))+rsi] |
| 828 | add rax,r9 |
| 829 | mov r10,rbx |
| 830 | imul r10,QWORD[((8-128))+rsi] |
| 831 | add r10,QWORD[8+rsp] |
| 832 | |
| 833 | mov r9,rax |
| 834 | imul eax,r8d |
| 835 | and eax,0x1fffffff |
| 836 | |
| 837 | mov r11,rbx |
| 838 | imul r11,QWORD[((16-128))+rsi] |
| 839 | add r11,QWORD[16+rsp] |
| 840 | |
| 841 | mov r12,rbx |
| 842 | imul r12,QWORD[((24-128))+rsi] |
| 843 | add r12,QWORD[24+rsp] |
| 844 | vpmuludq ymm0,ymm10,YMMWORD[((32-128))+rsi] |
| 845 | vmovd xmm11,eax |
| 846 | vpaddq ymm1,ymm1,ymm0 |
| 847 | vpmuludq ymm12,ymm10,YMMWORD[((64-128))+rsi] |
| 848 | vpbroadcastq ymm11,xmm11 |
| 849 | vpaddq ymm2,ymm2,ymm12 |
| 850 | vpmuludq ymm13,ymm10,YMMWORD[((96-128))+rsi] |
| 851 | vpand ymm3,ymm3,ymm15 |
| 852 | vpaddq ymm3,ymm3,ymm13 |
| 853 | vpmuludq ymm0,ymm10,YMMWORD[((128-128))+rsi] |
| 854 | vpaddq ymm4,ymm4,ymm0 |
| 855 | vpmuludq ymm12,ymm10,YMMWORD[((160-128))+rsi] |
| 856 | vpaddq ymm5,ymm5,ymm12 |
| 857 | vpmuludq ymm13,ymm10,YMMWORD[((192-128))+rsi] |
| 858 | vpaddq ymm6,ymm6,ymm13 |
| 859 | vpmuludq ymm0,ymm10,YMMWORD[((224-128))+rsi] |
| 860 | vpermq ymm9,ymm9,0x93 |
| 861 | vpaddq ymm7,ymm7,ymm0 |
| 862 | vpmuludq ymm12,ymm10,YMMWORD[((256-128))+rsi] |
| 863 | vpbroadcastq ymm10,QWORD[8+r13] |
| 864 | vpaddq ymm8,ymm8,ymm12 |
| 865 | |
| 866 | mov rdx,rax |
| 867 | imul rax,QWORD[((-128))+rcx] |
| 868 | add r9,rax |
| 869 | mov rax,rdx |
| 870 | imul rax,QWORD[((8-128))+rcx] |
| 871 | add r10,rax |
| 872 | mov rax,rdx |
| 873 | imul rax,QWORD[((16-128))+rcx] |
| 874 | add r11,rax |
| 875 | shr r9,29 |
| 876 | imul rdx,QWORD[((24-128))+rcx] |
| 877 | add r12,rdx |
| 878 | add r10,r9 |
| 879 | |
| 880 | vpmuludq ymm13,ymm11,YMMWORD[((32-128))+rcx] |
| 881 | vmovq rbx,xmm10 |
| 882 | vpaddq ymm1,ymm1,ymm13 |
| 883 | vpmuludq ymm0,ymm11,YMMWORD[((64-128))+rcx] |
| 884 | vpaddq ymm2,ymm2,ymm0 |
| 885 | vpmuludq ymm12,ymm11,YMMWORD[((96-128))+rcx] |
| 886 | vpaddq ymm3,ymm3,ymm12 |
| 887 | vpmuludq ymm13,ymm11,YMMWORD[((128-128))+rcx] |
| 888 | vpaddq ymm4,ymm4,ymm13 |
| 889 | vpmuludq ymm0,ymm11,YMMWORD[((160-128))+rcx] |
| 890 | vpaddq ymm5,ymm5,ymm0 |
| 891 | vpmuludq ymm12,ymm11,YMMWORD[((192-128))+rcx] |
| 892 | vpaddq ymm6,ymm6,ymm12 |
| 893 | vpmuludq ymm13,ymm11,YMMWORD[((224-128))+rcx] |
| 894 | vpblendd ymm9,ymm9,ymm14,3 |
| 895 | vpaddq ymm7,ymm7,ymm13 |
| 896 | vpmuludq ymm0,ymm11,YMMWORD[((256-128))+rcx] |
| 897 | vpaddq ymm3,ymm3,ymm9 |
| 898 | vpaddq ymm8,ymm8,ymm0 |
| 899 | |
| 900 | mov rax,rbx |
| 901 | imul rax,QWORD[((-128))+rsi] |
| 902 | add r10,rax |
| 903 | vmovdqu ymm12,YMMWORD[((-8+32-128))+rsi] |
| 904 | mov rax,rbx |
| 905 | imul rax,QWORD[((8-128))+rsi] |
| 906 | add r11,rax |
| 907 | vmovdqu ymm13,YMMWORD[((-8+64-128))+rsi] |
| 908 | |
| 909 | mov rax,r10 |
| 910 | imul eax,r8d |
| 911 | and eax,0x1fffffff |
| 912 | |
| 913 | imul rbx,QWORD[((16-128))+rsi] |
| 914 | add r12,rbx |
| 915 | vpmuludq ymm12,ymm12,ymm10 |
| 916 | vmovd xmm11,eax |
| 917 | vmovdqu ymm0,YMMWORD[((-8+96-128))+rsi] |
| 918 | vpaddq ymm1,ymm1,ymm12 |
| 919 | vpmuludq ymm13,ymm13,ymm10 |
| 920 | vpbroadcastq ymm11,xmm11 |
| 921 | vmovdqu ymm12,YMMWORD[((-8+128-128))+rsi] |
| 922 | vpaddq ymm2,ymm2,ymm13 |
| 923 | vpmuludq ymm0,ymm0,ymm10 |
| 924 | vmovdqu ymm13,YMMWORD[((-8+160-128))+rsi] |
| 925 | vpaddq ymm3,ymm3,ymm0 |
| 926 | vpmuludq ymm12,ymm12,ymm10 |
| 927 | vmovdqu ymm0,YMMWORD[((-8+192-128))+rsi] |
| 928 | vpaddq ymm4,ymm4,ymm12 |
| 929 | vpmuludq ymm13,ymm13,ymm10 |
| 930 | vmovdqu ymm12,YMMWORD[((-8+224-128))+rsi] |
| 931 | vpaddq ymm5,ymm5,ymm13 |
| 932 | vpmuludq ymm0,ymm0,ymm10 |
| 933 | vmovdqu ymm13,YMMWORD[((-8+256-128))+rsi] |
| 934 | vpaddq ymm6,ymm6,ymm0 |
| 935 | vpmuludq ymm12,ymm12,ymm10 |
| 936 | vmovdqu ymm9,YMMWORD[((-8+288-128))+rsi] |
| 937 | vpaddq ymm7,ymm7,ymm12 |
| 938 | vpmuludq ymm13,ymm13,ymm10 |
| 939 | vpaddq ymm8,ymm8,ymm13 |
| 940 | vpmuludq ymm9,ymm9,ymm10 |
| 941 | vpbroadcastq ymm10,QWORD[16+r13] |
| 942 | |
| 943 | mov rdx,rax |
| 944 | imul rax,QWORD[((-128))+rcx] |
| 945 | add r10,rax |
| 946 | vmovdqu ymm0,YMMWORD[((-8+32-128))+rcx] |
| 947 | mov rax,rdx |
| 948 | imul rax,QWORD[((8-128))+rcx] |
| 949 | add r11,rax |
| 950 | vmovdqu ymm12,YMMWORD[((-8+64-128))+rcx] |
| 951 | shr r10,29 |
| 952 | imul rdx,QWORD[((16-128))+rcx] |
| 953 | add r12,rdx |
| 954 | add r11,r10 |
| 955 | |
| 956 | vpmuludq ymm0,ymm0,ymm11 |
| 957 | vmovq rbx,xmm10 |
| 958 | vmovdqu ymm13,YMMWORD[((-8+96-128))+rcx] |
| 959 | vpaddq ymm1,ymm1,ymm0 |
| 960 | vpmuludq ymm12,ymm12,ymm11 |
| 961 | vmovdqu ymm0,YMMWORD[((-8+128-128))+rcx] |
| 962 | vpaddq ymm2,ymm2,ymm12 |
| 963 | vpmuludq ymm13,ymm13,ymm11 |
| 964 | vmovdqu ymm12,YMMWORD[((-8+160-128))+rcx] |
| 965 | vpaddq ymm3,ymm3,ymm13 |
| 966 | vpmuludq ymm0,ymm0,ymm11 |
| 967 | vmovdqu ymm13,YMMWORD[((-8+192-128))+rcx] |
| 968 | vpaddq ymm4,ymm4,ymm0 |
| 969 | vpmuludq ymm12,ymm12,ymm11 |
| 970 | vmovdqu ymm0,YMMWORD[((-8+224-128))+rcx] |
| 971 | vpaddq ymm5,ymm5,ymm12 |
| 972 | vpmuludq ymm13,ymm13,ymm11 |
| 973 | vmovdqu ymm12,YMMWORD[((-8+256-128))+rcx] |
| 974 | vpaddq ymm6,ymm6,ymm13 |
| 975 | vpmuludq ymm0,ymm0,ymm11 |
| 976 | vmovdqu ymm13,YMMWORD[((-8+288-128))+rcx] |
| 977 | vpaddq ymm7,ymm7,ymm0 |
| 978 | vpmuludq ymm12,ymm12,ymm11 |
| 979 | vpaddq ymm8,ymm8,ymm12 |
| 980 | vpmuludq ymm13,ymm13,ymm11 |
| 981 | vpaddq ymm9,ymm9,ymm13 |
| 982 | |
| 983 | vmovdqu ymm0,YMMWORD[((-16+32-128))+rsi] |
| 984 | mov rax,rbx |
| 985 | imul rax,QWORD[((-128))+rsi] |
| 986 | add rax,r11 |
| 987 | |
| 988 | vmovdqu ymm12,YMMWORD[((-16+64-128))+rsi] |
| 989 | mov r11,rax |
| 990 | imul eax,r8d |
| 991 | and eax,0x1fffffff |
| 992 | |
| 993 | imul rbx,QWORD[((8-128))+rsi] |
| 994 | add r12,rbx |
| 995 | vpmuludq ymm0,ymm0,ymm10 |
| 996 | vmovd xmm11,eax |
| 997 | vmovdqu ymm13,YMMWORD[((-16+96-128))+rsi] |
| 998 | vpaddq ymm1,ymm1,ymm0 |
| 999 | vpmuludq ymm12,ymm12,ymm10 |
| 1000 | vpbroadcastq ymm11,xmm11 |
| 1001 | vmovdqu ymm0,YMMWORD[((-16+128-128))+rsi] |
| 1002 | vpaddq ymm2,ymm2,ymm12 |
| 1003 | vpmuludq ymm13,ymm13,ymm10 |
| 1004 | vmovdqu ymm12,YMMWORD[((-16+160-128))+rsi] |
| 1005 | vpaddq ymm3,ymm3,ymm13 |
| 1006 | vpmuludq ymm0,ymm0,ymm10 |
| 1007 | vmovdqu ymm13,YMMWORD[((-16+192-128))+rsi] |
| 1008 | vpaddq ymm4,ymm4,ymm0 |
| 1009 | vpmuludq ymm12,ymm12,ymm10 |
| 1010 | vmovdqu ymm0,YMMWORD[((-16+224-128))+rsi] |
| 1011 | vpaddq ymm5,ymm5,ymm12 |
| 1012 | vpmuludq ymm13,ymm13,ymm10 |
| 1013 | vmovdqu ymm12,YMMWORD[((-16+256-128))+rsi] |
| 1014 | vpaddq ymm6,ymm6,ymm13 |
| 1015 | vpmuludq ymm0,ymm0,ymm10 |
| 1016 | vmovdqu ymm13,YMMWORD[((-16+288-128))+rsi] |
| 1017 | vpaddq ymm7,ymm7,ymm0 |
| 1018 | vpmuludq ymm12,ymm12,ymm10 |
| 1019 | vpaddq ymm8,ymm8,ymm12 |
| 1020 | vpmuludq ymm13,ymm13,ymm10 |
| 1021 | vpbroadcastq ymm10,QWORD[24+r13] |
| 1022 | vpaddq ymm9,ymm9,ymm13 |
| 1023 | |
| 1024 | vmovdqu ymm0,YMMWORD[((-16+32-128))+rcx] |
| 1025 | mov rdx,rax |
| 1026 | imul rax,QWORD[((-128))+rcx] |
| 1027 | add r11,rax |
| 1028 | vmovdqu ymm12,YMMWORD[((-16+64-128))+rcx] |
| 1029 | imul rdx,QWORD[((8-128))+rcx] |
| 1030 | add r12,rdx |
| 1031 | shr r11,29 |
| 1032 | |
| 1033 | vpmuludq ymm0,ymm0,ymm11 |
| 1034 | vmovq rbx,xmm10 |
| 1035 | vmovdqu ymm13,YMMWORD[((-16+96-128))+rcx] |
| 1036 | vpaddq ymm1,ymm1,ymm0 |
| 1037 | vpmuludq ymm12,ymm12,ymm11 |
| 1038 | vmovdqu ymm0,YMMWORD[((-16+128-128))+rcx] |
| 1039 | vpaddq ymm2,ymm2,ymm12 |
| 1040 | vpmuludq ymm13,ymm13,ymm11 |
| 1041 | vmovdqu ymm12,YMMWORD[((-16+160-128))+rcx] |
| 1042 | vpaddq ymm3,ymm3,ymm13 |
| 1043 | vpmuludq ymm0,ymm0,ymm11 |
| 1044 | vmovdqu ymm13,YMMWORD[((-16+192-128))+rcx] |
| 1045 | vpaddq ymm4,ymm4,ymm0 |
| 1046 | vpmuludq ymm12,ymm12,ymm11 |
| 1047 | vmovdqu ymm0,YMMWORD[((-16+224-128))+rcx] |
| 1048 | vpaddq ymm5,ymm5,ymm12 |
| 1049 | vpmuludq ymm13,ymm13,ymm11 |
| 1050 | vmovdqu ymm12,YMMWORD[((-16+256-128))+rcx] |
| 1051 | vpaddq ymm6,ymm6,ymm13 |
| 1052 | vpmuludq ymm0,ymm0,ymm11 |
| 1053 | vmovdqu ymm13,YMMWORD[((-16+288-128))+rcx] |
| 1054 | vpaddq ymm7,ymm7,ymm0 |
| 1055 | vpmuludq ymm12,ymm12,ymm11 |
| 1056 | vmovdqu ymm0,YMMWORD[((-24+32-128))+rsi] |
| 1057 | vpaddq ymm8,ymm8,ymm12 |
| 1058 | vpmuludq ymm13,ymm13,ymm11 |
| 1059 | vmovdqu ymm12,YMMWORD[((-24+64-128))+rsi] |
| 1060 | vpaddq ymm9,ymm9,ymm13 |
| 1061 | |
| 1062 | add r12,r11 |
| 1063 | imul rbx,QWORD[((-128))+rsi] |
| 1064 | add r12,rbx |
| 1065 | |
| 1066 | mov rax,r12 |
| 1067 | imul eax,r8d |
| 1068 | and eax,0x1fffffff |
| 1069 | |
| 1070 | vpmuludq ymm0,ymm0,ymm10 |
| 1071 | vmovd xmm11,eax |
| 1072 | vmovdqu ymm13,YMMWORD[((-24+96-128))+rsi] |
| 1073 | vpaddq ymm1,ymm1,ymm0 |
| 1074 | vpmuludq ymm12,ymm12,ymm10 |
| 1075 | vpbroadcastq ymm11,xmm11 |
| 1076 | vmovdqu ymm0,YMMWORD[((-24+128-128))+rsi] |
| 1077 | vpaddq ymm2,ymm2,ymm12 |
| 1078 | vpmuludq ymm13,ymm13,ymm10 |
| 1079 | vmovdqu ymm12,YMMWORD[((-24+160-128))+rsi] |
| 1080 | vpaddq ymm3,ymm3,ymm13 |
| 1081 | vpmuludq ymm0,ymm0,ymm10 |
| 1082 | vmovdqu ymm13,YMMWORD[((-24+192-128))+rsi] |
| 1083 | vpaddq ymm4,ymm4,ymm0 |
| 1084 | vpmuludq ymm12,ymm12,ymm10 |
| 1085 | vmovdqu ymm0,YMMWORD[((-24+224-128))+rsi] |
| 1086 | vpaddq ymm5,ymm5,ymm12 |
| 1087 | vpmuludq ymm13,ymm13,ymm10 |
| 1088 | vmovdqu ymm12,YMMWORD[((-24+256-128))+rsi] |
| 1089 | vpaddq ymm6,ymm6,ymm13 |
| 1090 | vpmuludq ymm0,ymm0,ymm10 |
| 1091 | vmovdqu ymm13,YMMWORD[((-24+288-128))+rsi] |
| 1092 | vpaddq ymm7,ymm7,ymm0 |
| 1093 | vpmuludq ymm12,ymm12,ymm10 |
| 1094 | vpaddq ymm8,ymm8,ymm12 |
| 1095 | vpmuludq ymm13,ymm13,ymm10 |
| 1096 | vpbroadcastq ymm10,QWORD[32+r13] |
| 1097 | vpaddq ymm9,ymm9,ymm13 |
| 1098 | add r13,32 |
| 1099 | |
| 1100 | vmovdqu ymm0,YMMWORD[((-24+32-128))+rcx] |
| 1101 | imul rax,QWORD[((-128))+rcx] |
| 1102 | add r12,rax |
| 1103 | shr r12,29 |
| 1104 | |
| 1105 | vmovdqu ymm12,YMMWORD[((-24+64-128))+rcx] |
| 1106 | vpmuludq ymm0,ymm0,ymm11 |
| 1107 | vmovq rbx,xmm10 |
| 1108 | vmovdqu ymm13,YMMWORD[((-24+96-128))+rcx] |
| 1109 | vpaddq ymm0,ymm1,ymm0 |
| 1110 | vpmuludq ymm12,ymm12,ymm11 |
| 1111 | vmovdqu YMMWORD[rsp],ymm0 |
| 1112 | vpaddq ymm1,ymm2,ymm12 |
| 1113 | vmovdqu ymm0,YMMWORD[((-24+128-128))+rcx] |
| 1114 | vpmuludq ymm13,ymm13,ymm11 |
| 1115 | vmovdqu ymm12,YMMWORD[((-24+160-128))+rcx] |
| 1116 | vpaddq ymm2,ymm3,ymm13 |
| 1117 | vpmuludq ymm0,ymm0,ymm11 |
| 1118 | vmovdqu ymm13,YMMWORD[((-24+192-128))+rcx] |
| 1119 | vpaddq ymm3,ymm4,ymm0 |
| 1120 | vpmuludq ymm12,ymm12,ymm11 |
| 1121 | vmovdqu ymm0,YMMWORD[((-24+224-128))+rcx] |
| 1122 | vpaddq ymm4,ymm5,ymm12 |
| 1123 | vpmuludq ymm13,ymm13,ymm11 |
| 1124 | vmovdqu ymm12,YMMWORD[((-24+256-128))+rcx] |
| 1125 | vpaddq ymm5,ymm6,ymm13 |
| 1126 | vpmuludq ymm0,ymm0,ymm11 |
| 1127 | vmovdqu ymm13,YMMWORD[((-24+288-128))+rcx] |
| 1128 | mov r9,r12 |
| 1129 | vpaddq ymm6,ymm7,ymm0 |
| 1130 | vpmuludq ymm12,ymm12,ymm11 |
| 1131 | add r9,QWORD[rsp] |
| 1132 | vpaddq ymm7,ymm8,ymm12 |
| 1133 | vpmuludq ymm13,ymm13,ymm11 |
| 1134 | vmovq xmm12,r12 |
| 1135 | vpaddq ymm8,ymm9,ymm13 |
| 1136 | |
| 1137 | dec r14d |
| 1138 | jnz NEAR $L$oop_mul_1024 |
| 1139 | vpermq ymm15,ymm15,0 |
| 1140 | vpaddq ymm0,ymm12,YMMWORD[rsp] |
| 1141 | |
| 1142 | vpsrlq ymm12,ymm0,29 |
| 1143 | vpand ymm0,ymm0,ymm15 |
| 1144 | vpsrlq ymm13,ymm1,29 |
| 1145 | vpand ymm1,ymm1,ymm15 |
| 1146 | vpsrlq ymm10,ymm2,29 |
| 1147 | vpermq ymm12,ymm12,0x93 |
| 1148 | vpand ymm2,ymm2,ymm15 |
| 1149 | vpsrlq ymm11,ymm3,29 |
| 1150 | vpermq ymm13,ymm13,0x93 |
| 1151 | vpand ymm3,ymm3,ymm15 |
| 1152 | |
| 1153 | vpblendd ymm9,ymm12,ymm14,3 |
| 1154 | vpermq ymm10,ymm10,0x93 |
| 1155 | vpblendd ymm12,ymm13,ymm12,3 |
| 1156 | vpermq ymm11,ymm11,0x93 |
| 1157 | vpaddq ymm0,ymm0,ymm9 |
| 1158 | vpblendd ymm13,ymm10,ymm13,3 |
| 1159 | vpaddq ymm1,ymm1,ymm12 |
| 1160 | vpblendd ymm10,ymm11,ymm10,3 |
| 1161 | vpaddq ymm2,ymm2,ymm13 |
| 1162 | vpblendd ymm11,ymm14,ymm11,3 |
| 1163 | vpaddq ymm3,ymm3,ymm10 |
| 1164 | vpaddq ymm4,ymm4,ymm11 |
| 1165 | |
| 1166 | vpsrlq ymm12,ymm0,29 |
| 1167 | vpand ymm0,ymm0,ymm15 |
| 1168 | vpsrlq ymm13,ymm1,29 |
| 1169 | vpand ymm1,ymm1,ymm15 |
| 1170 | vpsrlq ymm10,ymm2,29 |
| 1171 | vpermq ymm12,ymm12,0x93 |
| 1172 | vpand ymm2,ymm2,ymm15 |
| 1173 | vpsrlq ymm11,ymm3,29 |
| 1174 | vpermq ymm13,ymm13,0x93 |
| 1175 | vpand ymm3,ymm3,ymm15 |
| 1176 | vpermq ymm10,ymm10,0x93 |
| 1177 | |
| 1178 | vpblendd ymm9,ymm12,ymm14,3 |
| 1179 | vpermq ymm11,ymm11,0x93 |
| 1180 | vpblendd ymm12,ymm13,ymm12,3 |
| 1181 | vpaddq ymm0,ymm0,ymm9 |
| 1182 | vpblendd ymm13,ymm10,ymm13,3 |
| 1183 | vpaddq ymm1,ymm1,ymm12 |
| 1184 | vpblendd ymm10,ymm11,ymm10,3 |
| 1185 | vpaddq ymm2,ymm2,ymm13 |
| 1186 | vpblendd ymm11,ymm14,ymm11,3 |
| 1187 | vpaddq ymm3,ymm3,ymm10 |
| 1188 | vpaddq ymm4,ymm4,ymm11 |
| 1189 | |
| 1190 | vmovdqu YMMWORD[(0-128)+rdi],ymm0 |
| 1191 | vmovdqu YMMWORD[(32-128)+rdi],ymm1 |
| 1192 | vmovdqu YMMWORD[(64-128)+rdi],ymm2 |
| 1193 | vmovdqu YMMWORD[(96-128)+rdi],ymm3 |
| 1194 | vpsrlq ymm12,ymm4,29 |
| 1195 | vpand ymm4,ymm4,ymm15 |
| 1196 | vpsrlq ymm13,ymm5,29 |
| 1197 | vpand ymm5,ymm5,ymm15 |
| 1198 | vpsrlq ymm10,ymm6,29 |
| 1199 | vpermq ymm12,ymm12,0x93 |
| 1200 | vpand ymm6,ymm6,ymm15 |
| 1201 | vpsrlq ymm11,ymm7,29 |
| 1202 | vpermq ymm13,ymm13,0x93 |
| 1203 | vpand ymm7,ymm7,ymm15 |
| 1204 | vpsrlq ymm0,ymm8,29 |
| 1205 | vpermq ymm10,ymm10,0x93 |
| 1206 | vpand ymm8,ymm8,ymm15 |
| 1207 | vpermq ymm11,ymm11,0x93 |
| 1208 | |
| 1209 | vpblendd ymm9,ymm12,ymm14,3 |
| 1210 | vpermq ymm0,ymm0,0x93 |
| 1211 | vpblendd ymm12,ymm13,ymm12,3 |
| 1212 | vpaddq ymm4,ymm4,ymm9 |
| 1213 | vpblendd ymm13,ymm10,ymm13,3 |
| 1214 | vpaddq ymm5,ymm5,ymm12 |
| 1215 | vpblendd ymm10,ymm11,ymm10,3 |
| 1216 | vpaddq ymm6,ymm6,ymm13 |
| 1217 | vpblendd ymm11,ymm0,ymm11,3 |
| 1218 | vpaddq ymm7,ymm7,ymm10 |
| 1219 | vpaddq ymm8,ymm8,ymm11 |
| 1220 | |
| 1221 | vpsrlq ymm12,ymm4,29 |
| 1222 | vpand ymm4,ymm4,ymm15 |
| 1223 | vpsrlq ymm13,ymm5,29 |
| 1224 | vpand ymm5,ymm5,ymm15 |
| 1225 | vpsrlq ymm10,ymm6,29 |
| 1226 | vpermq ymm12,ymm12,0x93 |
| 1227 | vpand ymm6,ymm6,ymm15 |
| 1228 | vpsrlq ymm11,ymm7,29 |
| 1229 | vpermq ymm13,ymm13,0x93 |
| 1230 | vpand ymm7,ymm7,ymm15 |
| 1231 | vpsrlq ymm0,ymm8,29 |
| 1232 | vpermq ymm10,ymm10,0x93 |
| 1233 | vpand ymm8,ymm8,ymm15 |
| 1234 | vpermq ymm11,ymm11,0x93 |
| 1235 | |
| 1236 | vpblendd ymm9,ymm12,ymm14,3 |
| 1237 | vpermq ymm0,ymm0,0x93 |
| 1238 | vpblendd ymm12,ymm13,ymm12,3 |
| 1239 | vpaddq ymm4,ymm4,ymm9 |
| 1240 | vpblendd ymm13,ymm10,ymm13,3 |
| 1241 | vpaddq ymm5,ymm5,ymm12 |
| 1242 | vpblendd ymm10,ymm11,ymm10,3 |
| 1243 | vpaddq ymm6,ymm6,ymm13 |
| 1244 | vpblendd ymm11,ymm0,ymm11,3 |
| 1245 | vpaddq ymm7,ymm7,ymm10 |
| 1246 | vpaddq ymm8,ymm8,ymm11 |
| 1247 | |
| 1248 | vmovdqu YMMWORD[(128-128)+rdi],ymm4 |
| 1249 | vmovdqu YMMWORD[(160-128)+rdi],ymm5 |
| 1250 | vmovdqu YMMWORD[(192-128)+rdi],ymm6 |
| 1251 | vmovdqu YMMWORD[(224-128)+rdi],ymm7 |
| 1252 | vmovdqu YMMWORD[(256-128)+rdi],ymm8 |
| 1253 | vzeroupper |
| 1254 | |
| 1255 | mov rax,rbp |
| 1256 | |
| 1257 | $L$mul_1024_in_tail: |
| 1258 | movaps xmm6,XMMWORD[((-216))+rax] |
| 1259 | movaps xmm7,XMMWORD[((-200))+rax] |
| 1260 | movaps xmm8,XMMWORD[((-184))+rax] |
| 1261 | movaps xmm9,XMMWORD[((-168))+rax] |
| 1262 | movaps xmm10,XMMWORD[((-152))+rax] |
| 1263 | movaps xmm11,XMMWORD[((-136))+rax] |
| 1264 | movaps xmm12,XMMWORD[((-120))+rax] |
| 1265 | movaps xmm13,XMMWORD[((-104))+rax] |
| 1266 | movaps xmm14,XMMWORD[((-88))+rax] |
| 1267 | movaps xmm15,XMMWORD[((-72))+rax] |
| 1268 | mov r15,QWORD[((-48))+rax] |
| 1269 | |
| 1270 | mov r14,QWORD[((-40))+rax] |
| 1271 | |
| 1272 | mov r13,QWORD[((-32))+rax] |
| 1273 | |
| 1274 | mov r12,QWORD[((-24))+rax] |
| 1275 | |
| 1276 | mov rbp,QWORD[((-16))+rax] |
| 1277 | |
| 1278 | mov rbx,QWORD[((-8))+rax] |
| 1279 | |
| 1280 | lea rsp,[rax] |
| 1281 | |
| 1282 | $L$mul_1024_epilogue: |
| 1283 | mov rdi,QWORD[8+rsp] ;WIN64 epilogue |
| 1284 | mov rsi,QWORD[16+rsp] |
| 1285 | DB 0F3h,0C3h ;repret |
| 1286 | |
| 1287 | $L$SEH_end_rsaz_1024_mul_avx2: |
| 1288 | global rsaz_1024_red2norm_avx2 |
| 1289 | |
| 1290 | ALIGN 32 |
| 1291 | rsaz_1024_red2norm_avx2: |
| 1292 | sub rdx,-128 |
| 1293 | xor rax,rax |
| 1294 | mov r8,QWORD[((-128))+rdx] |
| 1295 | mov r9,QWORD[((-120))+rdx] |
| 1296 | mov r10,QWORD[((-112))+rdx] |
| 1297 | shl r8,0 |
| 1298 | shl r9,29 |
| 1299 | mov r11,r10 |
| 1300 | shl r10,58 |
| 1301 | shr r11,6 |
| 1302 | add rax,r8 |
| 1303 | add rax,r9 |
| 1304 | add rax,r10 |
| 1305 | adc r11,0 |
| 1306 | mov QWORD[rcx],rax |
| 1307 | mov rax,r11 |
| 1308 | mov r8,QWORD[((-104))+rdx] |
| 1309 | mov r9,QWORD[((-96))+rdx] |
| 1310 | shl r8,23 |
| 1311 | mov r10,r9 |
| 1312 | shl r9,52 |
| 1313 | shr r10,12 |
| 1314 | add rax,r8 |
| 1315 | add rax,r9 |
| 1316 | adc r10,0 |
| 1317 | mov QWORD[8+rcx],rax |
| 1318 | mov rax,r10 |
| 1319 | mov r11,QWORD[((-88))+rdx] |
| 1320 | mov r8,QWORD[((-80))+rdx] |
| 1321 | shl r11,17 |
| 1322 | mov r9,r8 |
| 1323 | shl r8,46 |
| 1324 | shr r9,18 |
| 1325 | add rax,r11 |
| 1326 | add rax,r8 |
| 1327 | adc r9,0 |
| 1328 | mov QWORD[16+rcx],rax |
| 1329 | mov rax,r9 |
| 1330 | mov r10,QWORD[((-72))+rdx] |
| 1331 | mov r11,QWORD[((-64))+rdx] |
| 1332 | shl r10,11 |
| 1333 | mov r8,r11 |
| 1334 | shl r11,40 |
| 1335 | shr r8,24 |
| 1336 | add rax,r10 |
| 1337 | add rax,r11 |
| 1338 | adc r8,0 |
| 1339 | mov QWORD[24+rcx],rax |
| 1340 | mov rax,r8 |
| 1341 | mov r9,QWORD[((-56))+rdx] |
| 1342 | mov r10,QWORD[((-48))+rdx] |
| 1343 | mov r11,QWORD[((-40))+rdx] |
| 1344 | shl r9,5 |
| 1345 | shl r10,34 |
| 1346 | mov r8,r11 |
| 1347 | shl r11,63 |
| 1348 | shr r8,1 |
| 1349 | add rax,r9 |
| 1350 | add rax,r10 |
| 1351 | add rax,r11 |
| 1352 | adc r8,0 |
| 1353 | mov QWORD[32+rcx],rax |
| 1354 | mov rax,r8 |
| 1355 | mov r9,QWORD[((-32))+rdx] |
| 1356 | mov r10,QWORD[((-24))+rdx] |
| 1357 | shl r9,28 |
| 1358 | mov r11,r10 |
| 1359 | shl r10,57 |
| 1360 | shr r11,7 |
| 1361 | add rax,r9 |
| 1362 | add rax,r10 |
| 1363 | adc r11,0 |
| 1364 | mov QWORD[40+rcx],rax |
| 1365 | mov rax,r11 |
| 1366 | mov r8,QWORD[((-16))+rdx] |
| 1367 | mov r9,QWORD[((-8))+rdx] |
| 1368 | shl r8,22 |
| 1369 | mov r10,r9 |
| 1370 | shl r9,51 |
| 1371 | shr r10,13 |
| 1372 | add rax,r8 |
| 1373 | add rax,r9 |
| 1374 | adc r10,0 |
| 1375 | mov QWORD[48+rcx],rax |
| 1376 | mov rax,r10 |
| 1377 | mov r11,QWORD[rdx] |
| 1378 | mov r8,QWORD[8+rdx] |
| 1379 | shl r11,16 |
| 1380 | mov r9,r8 |
| 1381 | shl r8,45 |
| 1382 | shr r9,19 |
| 1383 | add rax,r11 |
| 1384 | add rax,r8 |
| 1385 | adc r9,0 |
| 1386 | mov QWORD[56+rcx],rax |
| 1387 | mov rax,r9 |
| 1388 | mov r10,QWORD[16+rdx] |
| 1389 | mov r11,QWORD[24+rdx] |
| 1390 | shl r10,10 |
| 1391 | mov r8,r11 |
| 1392 | shl r11,39 |
| 1393 | shr r8,25 |
| 1394 | add rax,r10 |
| 1395 | add rax,r11 |
| 1396 | adc r8,0 |
| 1397 | mov QWORD[64+rcx],rax |
| 1398 | mov rax,r8 |
| 1399 | mov r9,QWORD[32+rdx] |
| 1400 | mov r10,QWORD[40+rdx] |
| 1401 | mov r11,QWORD[48+rdx] |
| 1402 | shl r9,4 |
| 1403 | shl r10,33 |
| 1404 | mov r8,r11 |
| 1405 | shl r11,62 |
| 1406 | shr r8,2 |
| 1407 | add rax,r9 |
| 1408 | add rax,r10 |
| 1409 | add rax,r11 |
| 1410 | adc r8,0 |
| 1411 | mov QWORD[72+rcx],rax |
| 1412 | mov rax,r8 |
| 1413 | mov r9,QWORD[56+rdx] |
| 1414 | mov r10,QWORD[64+rdx] |
| 1415 | shl r9,27 |
| 1416 | mov r11,r10 |
| 1417 | shl r10,56 |
| 1418 | shr r11,8 |
| 1419 | add rax,r9 |
| 1420 | add rax,r10 |
| 1421 | adc r11,0 |
| 1422 | mov QWORD[80+rcx],rax |
| 1423 | mov rax,r11 |
| 1424 | mov r8,QWORD[72+rdx] |
| 1425 | mov r9,QWORD[80+rdx] |
| 1426 | shl r8,21 |
| 1427 | mov r10,r9 |
| 1428 | shl r9,50 |
| 1429 | shr r10,14 |
| 1430 | add rax,r8 |
| 1431 | add rax,r9 |
| 1432 | adc r10,0 |
| 1433 | mov QWORD[88+rcx],rax |
| 1434 | mov rax,r10 |
| 1435 | mov r11,QWORD[88+rdx] |
| 1436 | mov r8,QWORD[96+rdx] |
| 1437 | shl r11,15 |
| 1438 | mov r9,r8 |
| 1439 | shl r8,44 |
| 1440 | shr r9,20 |
| 1441 | add rax,r11 |
| 1442 | add rax,r8 |
| 1443 | adc r9,0 |
| 1444 | mov QWORD[96+rcx],rax |
| 1445 | mov rax,r9 |
| 1446 | mov r10,QWORD[104+rdx] |
| 1447 | mov r11,QWORD[112+rdx] |
| 1448 | shl r10,9 |
| 1449 | mov r8,r11 |
| 1450 | shl r11,38 |
| 1451 | shr r8,26 |
| 1452 | add rax,r10 |
| 1453 | add rax,r11 |
| 1454 | adc r8,0 |
| 1455 | mov QWORD[104+rcx],rax |
| 1456 | mov rax,r8 |
| 1457 | mov r9,QWORD[120+rdx] |
| 1458 | mov r10,QWORD[128+rdx] |
| 1459 | mov r11,QWORD[136+rdx] |
| 1460 | shl r9,3 |
| 1461 | shl r10,32 |
| 1462 | mov r8,r11 |
| 1463 | shl r11,61 |
| 1464 | shr r8,3 |
| 1465 | add rax,r9 |
| 1466 | add rax,r10 |
| 1467 | add rax,r11 |
| 1468 | adc r8,0 |
| 1469 | mov QWORD[112+rcx],rax |
| 1470 | mov rax,r8 |
| 1471 | mov r9,QWORD[144+rdx] |
| 1472 | mov r10,QWORD[152+rdx] |
| 1473 | shl r9,26 |
| 1474 | mov r11,r10 |
| 1475 | shl r10,55 |
| 1476 | shr r11,9 |
| 1477 | add rax,r9 |
| 1478 | add rax,r10 |
| 1479 | adc r11,0 |
| 1480 | mov QWORD[120+rcx],rax |
| 1481 | mov rax,r11 |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 1482 | DB 0F3h,0C3h ;repret |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 1483 | |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 1484 | |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 1485 | global rsaz_1024_norm2red_avx2 |
Robert Sloan | 1c9db53 | 2017-03-13 08:03:59 -0700 | [diff] [blame] | 1486 | |
| 1487 | ALIGN 32 |
| 1488 | rsaz_1024_norm2red_avx2: |
| 1489 | sub rcx,-128 |
| 1490 | mov r8,QWORD[rdx] |
| 1491 | mov eax,0x1fffffff |
| 1492 | mov r9,QWORD[8+rdx] |
| 1493 | mov r11,r8 |
| 1494 | shr r11,0 |
| 1495 | and r11,rax |
| 1496 | mov QWORD[((-128))+rcx],r11 |
| 1497 | mov r10,r8 |
| 1498 | shr r10,29 |
| 1499 | and r10,rax |
| 1500 | mov QWORD[((-120))+rcx],r10 |
| 1501 | shrd r8,r9,58 |
| 1502 | and r8,rax |
| 1503 | mov QWORD[((-112))+rcx],r8 |
| 1504 | mov r10,QWORD[16+rdx] |
| 1505 | mov r8,r9 |
| 1506 | shr r8,23 |
| 1507 | and r8,rax |
| 1508 | mov QWORD[((-104))+rcx],r8 |
| 1509 | shrd r9,r10,52 |
| 1510 | and r9,rax |
| 1511 | mov QWORD[((-96))+rcx],r9 |
| 1512 | mov r11,QWORD[24+rdx] |
| 1513 | mov r9,r10 |
| 1514 | shr r9,17 |
| 1515 | and r9,rax |
| 1516 | mov QWORD[((-88))+rcx],r9 |
| 1517 | shrd r10,r11,46 |
| 1518 | and r10,rax |
| 1519 | mov QWORD[((-80))+rcx],r10 |
| 1520 | mov r8,QWORD[32+rdx] |
| 1521 | mov r10,r11 |
| 1522 | shr r10,11 |
| 1523 | and r10,rax |
| 1524 | mov QWORD[((-72))+rcx],r10 |
| 1525 | shrd r11,r8,40 |
| 1526 | and r11,rax |
| 1527 | mov QWORD[((-64))+rcx],r11 |
| 1528 | mov r9,QWORD[40+rdx] |
| 1529 | mov r11,r8 |
| 1530 | shr r11,5 |
| 1531 | and r11,rax |
| 1532 | mov QWORD[((-56))+rcx],r11 |
| 1533 | mov r10,r8 |
| 1534 | shr r10,34 |
| 1535 | and r10,rax |
| 1536 | mov QWORD[((-48))+rcx],r10 |
| 1537 | shrd r8,r9,63 |
| 1538 | and r8,rax |
| 1539 | mov QWORD[((-40))+rcx],r8 |
| 1540 | mov r10,QWORD[48+rdx] |
| 1541 | mov r8,r9 |
| 1542 | shr r8,28 |
| 1543 | and r8,rax |
| 1544 | mov QWORD[((-32))+rcx],r8 |
| 1545 | shrd r9,r10,57 |
| 1546 | and r9,rax |
| 1547 | mov QWORD[((-24))+rcx],r9 |
| 1548 | mov r11,QWORD[56+rdx] |
| 1549 | mov r9,r10 |
| 1550 | shr r9,22 |
| 1551 | and r9,rax |
| 1552 | mov QWORD[((-16))+rcx],r9 |
| 1553 | shrd r10,r11,51 |
| 1554 | and r10,rax |
| 1555 | mov QWORD[((-8))+rcx],r10 |
| 1556 | mov r8,QWORD[64+rdx] |
| 1557 | mov r10,r11 |
| 1558 | shr r10,16 |
| 1559 | and r10,rax |
| 1560 | mov QWORD[rcx],r10 |
| 1561 | shrd r11,r8,45 |
| 1562 | and r11,rax |
| 1563 | mov QWORD[8+rcx],r11 |
| 1564 | mov r9,QWORD[72+rdx] |
| 1565 | mov r11,r8 |
| 1566 | shr r11,10 |
| 1567 | and r11,rax |
| 1568 | mov QWORD[16+rcx],r11 |
| 1569 | shrd r8,r9,39 |
| 1570 | and r8,rax |
| 1571 | mov QWORD[24+rcx],r8 |
| 1572 | mov r10,QWORD[80+rdx] |
| 1573 | mov r8,r9 |
| 1574 | shr r8,4 |
| 1575 | and r8,rax |
| 1576 | mov QWORD[32+rcx],r8 |
| 1577 | mov r11,r9 |
| 1578 | shr r11,33 |
| 1579 | and r11,rax |
| 1580 | mov QWORD[40+rcx],r11 |
| 1581 | shrd r9,r10,62 |
| 1582 | and r9,rax |
| 1583 | mov QWORD[48+rcx],r9 |
| 1584 | mov r11,QWORD[88+rdx] |
| 1585 | mov r9,r10 |
| 1586 | shr r9,27 |
| 1587 | and r9,rax |
| 1588 | mov QWORD[56+rcx],r9 |
| 1589 | shrd r10,r11,56 |
| 1590 | and r10,rax |
| 1591 | mov QWORD[64+rcx],r10 |
| 1592 | mov r8,QWORD[96+rdx] |
| 1593 | mov r10,r11 |
| 1594 | shr r10,21 |
| 1595 | and r10,rax |
| 1596 | mov QWORD[72+rcx],r10 |
| 1597 | shrd r11,r8,50 |
| 1598 | and r11,rax |
| 1599 | mov QWORD[80+rcx],r11 |
| 1600 | mov r9,QWORD[104+rdx] |
| 1601 | mov r11,r8 |
| 1602 | shr r11,15 |
| 1603 | and r11,rax |
| 1604 | mov QWORD[88+rcx],r11 |
| 1605 | shrd r8,r9,44 |
| 1606 | and r8,rax |
| 1607 | mov QWORD[96+rcx],r8 |
| 1608 | mov r10,QWORD[112+rdx] |
| 1609 | mov r8,r9 |
| 1610 | shr r8,9 |
| 1611 | and r8,rax |
| 1612 | mov QWORD[104+rcx],r8 |
| 1613 | shrd r9,r10,38 |
| 1614 | and r9,rax |
| 1615 | mov QWORD[112+rcx],r9 |
| 1616 | mov r11,QWORD[120+rdx] |
| 1617 | mov r9,r10 |
| 1618 | shr r9,3 |
| 1619 | and r9,rax |
| 1620 | mov QWORD[120+rcx],r9 |
| 1621 | mov r8,r10 |
| 1622 | shr r8,32 |
| 1623 | and r8,rax |
| 1624 | mov QWORD[128+rcx],r8 |
| 1625 | shrd r10,r11,61 |
| 1626 | and r10,rax |
| 1627 | mov QWORD[136+rcx],r10 |
| 1628 | xor r8,r8 |
| 1629 | mov r10,r11 |
| 1630 | shr r10,26 |
| 1631 | and r10,rax |
| 1632 | mov QWORD[144+rcx],r10 |
| 1633 | shrd r11,r8,55 |
| 1634 | and r11,rax |
| 1635 | mov QWORD[152+rcx],r11 |
| 1636 | mov QWORD[160+rcx],r8 |
| 1637 | mov QWORD[168+rcx],r8 |
| 1638 | mov QWORD[176+rcx],r8 |
| 1639 | mov QWORD[184+rcx],r8 |
| 1640 | DB 0F3h,0C3h ;repret |
| 1641 | |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 1642 | global rsaz_1024_scatter5_avx2 |
Robert Sloan | 1c9db53 | 2017-03-13 08:03:59 -0700 | [diff] [blame] | 1643 | |
| 1644 | ALIGN 32 |
| 1645 | rsaz_1024_scatter5_avx2: |
| 1646 | vzeroupper |
| 1647 | vmovdqu ymm5,YMMWORD[$L$scatter_permd] |
| 1648 | shl r8d,4 |
| 1649 | lea rcx,[r8*1+rcx] |
| 1650 | mov eax,9 |
| 1651 | jmp NEAR $L$oop_scatter_1024 |
| 1652 | |
| 1653 | ALIGN 32 |
| 1654 | $L$oop_scatter_1024: |
| 1655 | vmovdqu ymm0,YMMWORD[rdx] |
| 1656 | lea rdx,[32+rdx] |
| 1657 | vpermd ymm0,ymm5,ymm0 |
| 1658 | vmovdqu XMMWORD[rcx],xmm0 |
| 1659 | lea rcx,[512+rcx] |
| 1660 | dec eax |
| 1661 | jnz NEAR $L$oop_scatter_1024 |
| 1662 | |
| 1663 | vzeroupper |
| 1664 | DB 0F3h,0C3h ;repret |
| 1665 | |
| 1666 | |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 1667 | global rsaz_1024_gather5_avx2 |
| 1668 | |
Robert Sloan | 1c9db53 | 2017-03-13 08:03:59 -0700 | [diff] [blame] | 1669 | ALIGN 32 |
Adam Langley | e9ada86 | 2015-05-11 17:20:37 -0700 | [diff] [blame] | 1670 | rsaz_1024_gather5_avx2: |
Robert Sloan | 1c9db53 | 2017-03-13 08:03:59 -0700 | [diff] [blame] | 1671 | |
| 1672 | vzeroupper |
| 1673 | mov r11,rsp |
| 1674 | |
| 1675 | lea rax,[((-136))+rsp] |
| 1676 | $L$SEH_begin_rsaz_1024_gather5: |
| 1677 | |
| 1678 | DB 0x48,0x8d,0x60,0xe0 |
| 1679 | DB 0xc5,0xf8,0x29,0x70,0xe0 |
| 1680 | DB 0xc5,0xf8,0x29,0x78,0xf0 |
| 1681 | DB 0xc5,0x78,0x29,0x40,0x00 |
| 1682 | DB 0xc5,0x78,0x29,0x48,0x10 |
| 1683 | DB 0xc5,0x78,0x29,0x50,0x20 |
| 1684 | DB 0xc5,0x78,0x29,0x58,0x30 |
| 1685 | DB 0xc5,0x78,0x29,0x60,0x40 |
| 1686 | DB 0xc5,0x78,0x29,0x68,0x50 |
| 1687 | DB 0xc5,0x78,0x29,0x70,0x60 |
| 1688 | DB 0xc5,0x78,0x29,0x78,0x70 |
| 1689 | lea rsp,[((-256))+rsp] |
| 1690 | and rsp,-32 |
| 1691 | lea r10,[$L$inc] |
| 1692 | lea rax,[((-128))+rsp] |
| 1693 | |
| 1694 | vmovd xmm4,r8d |
| 1695 | vmovdqa ymm0,YMMWORD[r10] |
| 1696 | vmovdqa ymm1,YMMWORD[32+r10] |
| 1697 | vmovdqa ymm5,YMMWORD[64+r10] |
| 1698 | vpbroadcastd ymm4,xmm4 |
| 1699 | |
| 1700 | vpaddd ymm2,ymm0,ymm5 |
| 1701 | vpcmpeqd ymm0,ymm0,ymm4 |
| 1702 | vpaddd ymm3,ymm1,ymm5 |
| 1703 | vpcmpeqd ymm1,ymm1,ymm4 |
| 1704 | vmovdqa YMMWORD[(0+128)+rax],ymm0 |
| 1705 | vpaddd ymm0,ymm2,ymm5 |
| 1706 | vpcmpeqd ymm2,ymm2,ymm4 |
| 1707 | vmovdqa YMMWORD[(32+128)+rax],ymm1 |
| 1708 | vpaddd ymm1,ymm3,ymm5 |
| 1709 | vpcmpeqd ymm3,ymm3,ymm4 |
| 1710 | vmovdqa YMMWORD[(64+128)+rax],ymm2 |
| 1711 | vpaddd ymm2,ymm0,ymm5 |
| 1712 | vpcmpeqd ymm0,ymm0,ymm4 |
| 1713 | vmovdqa YMMWORD[(96+128)+rax],ymm3 |
| 1714 | vpaddd ymm3,ymm1,ymm5 |
| 1715 | vpcmpeqd ymm1,ymm1,ymm4 |
| 1716 | vmovdqa YMMWORD[(128+128)+rax],ymm0 |
| 1717 | vpaddd ymm8,ymm2,ymm5 |
| 1718 | vpcmpeqd ymm2,ymm2,ymm4 |
| 1719 | vmovdqa YMMWORD[(160+128)+rax],ymm1 |
| 1720 | vpaddd ymm9,ymm3,ymm5 |
| 1721 | vpcmpeqd ymm3,ymm3,ymm4 |
| 1722 | vmovdqa YMMWORD[(192+128)+rax],ymm2 |
| 1723 | vpaddd ymm10,ymm8,ymm5 |
| 1724 | vpcmpeqd ymm8,ymm8,ymm4 |
| 1725 | vmovdqa YMMWORD[(224+128)+rax],ymm3 |
| 1726 | vpaddd ymm11,ymm9,ymm5 |
| 1727 | vpcmpeqd ymm9,ymm9,ymm4 |
| 1728 | vpaddd ymm12,ymm10,ymm5 |
| 1729 | vpcmpeqd ymm10,ymm10,ymm4 |
| 1730 | vpaddd ymm13,ymm11,ymm5 |
| 1731 | vpcmpeqd ymm11,ymm11,ymm4 |
| 1732 | vpaddd ymm14,ymm12,ymm5 |
| 1733 | vpcmpeqd ymm12,ymm12,ymm4 |
| 1734 | vpaddd ymm15,ymm13,ymm5 |
| 1735 | vpcmpeqd ymm13,ymm13,ymm4 |
| 1736 | vpcmpeqd ymm14,ymm14,ymm4 |
| 1737 | vpcmpeqd ymm15,ymm15,ymm4 |
| 1738 | |
| 1739 | vmovdqa ymm7,YMMWORD[((-32))+r10] |
| 1740 | lea rdx,[128+rdx] |
| 1741 | mov r8d,9 |
| 1742 | |
| 1743 | $L$oop_gather_1024: |
| 1744 | vmovdqa ymm0,YMMWORD[((0-128))+rdx] |
| 1745 | vmovdqa ymm1,YMMWORD[((32-128))+rdx] |
| 1746 | vmovdqa ymm2,YMMWORD[((64-128))+rdx] |
| 1747 | vmovdqa ymm3,YMMWORD[((96-128))+rdx] |
| 1748 | vpand ymm0,ymm0,YMMWORD[((0+128))+rax] |
| 1749 | vpand ymm1,ymm1,YMMWORD[((32+128))+rax] |
| 1750 | vpand ymm2,ymm2,YMMWORD[((64+128))+rax] |
| 1751 | vpor ymm4,ymm1,ymm0 |
| 1752 | vpand ymm3,ymm3,YMMWORD[((96+128))+rax] |
| 1753 | vmovdqa ymm0,YMMWORD[((128-128))+rdx] |
| 1754 | vmovdqa ymm1,YMMWORD[((160-128))+rdx] |
| 1755 | vpor ymm5,ymm3,ymm2 |
| 1756 | vmovdqa ymm2,YMMWORD[((192-128))+rdx] |
| 1757 | vmovdqa ymm3,YMMWORD[((224-128))+rdx] |
| 1758 | vpand ymm0,ymm0,YMMWORD[((128+128))+rax] |
| 1759 | vpand ymm1,ymm1,YMMWORD[((160+128))+rax] |
| 1760 | vpand ymm2,ymm2,YMMWORD[((192+128))+rax] |
| 1761 | vpor ymm4,ymm4,ymm0 |
| 1762 | vpand ymm3,ymm3,YMMWORD[((224+128))+rax] |
| 1763 | vpand ymm0,ymm8,YMMWORD[((256-128))+rdx] |
| 1764 | vpor ymm5,ymm5,ymm1 |
| 1765 | vpand ymm1,ymm9,YMMWORD[((288-128))+rdx] |
| 1766 | vpor ymm4,ymm4,ymm2 |
| 1767 | vpand ymm2,ymm10,YMMWORD[((320-128))+rdx] |
| 1768 | vpor ymm5,ymm5,ymm3 |
| 1769 | vpand ymm3,ymm11,YMMWORD[((352-128))+rdx] |
| 1770 | vpor ymm4,ymm4,ymm0 |
| 1771 | vpand ymm0,ymm12,YMMWORD[((384-128))+rdx] |
| 1772 | vpor ymm5,ymm5,ymm1 |
| 1773 | vpand ymm1,ymm13,YMMWORD[((416-128))+rdx] |
| 1774 | vpor ymm4,ymm4,ymm2 |
| 1775 | vpand ymm2,ymm14,YMMWORD[((448-128))+rdx] |
| 1776 | vpor ymm5,ymm5,ymm3 |
| 1777 | vpand ymm3,ymm15,YMMWORD[((480-128))+rdx] |
| 1778 | lea rdx,[512+rdx] |
| 1779 | vpor ymm4,ymm4,ymm0 |
| 1780 | vpor ymm5,ymm5,ymm1 |
| 1781 | vpor ymm4,ymm4,ymm2 |
| 1782 | vpor ymm5,ymm5,ymm3 |
| 1783 | |
| 1784 | vpor ymm4,ymm4,ymm5 |
| 1785 | vextracti128 xmm5,ymm4,1 |
| 1786 | vpor xmm5,xmm5,xmm4 |
| 1787 | vpermd ymm5,ymm7,ymm5 |
| 1788 | vmovdqu YMMWORD[rcx],ymm5 |
| 1789 | lea rcx,[32+rcx] |
| 1790 | dec r8d |
| 1791 | jnz NEAR $L$oop_gather_1024 |
| 1792 | |
| 1793 | vpxor ymm0,ymm0,ymm0 |
| 1794 | vmovdqu YMMWORD[rcx],ymm0 |
| 1795 | vzeroupper |
| 1796 | movaps xmm6,XMMWORD[((-168))+r11] |
| 1797 | movaps xmm7,XMMWORD[((-152))+r11] |
| 1798 | movaps xmm8,XMMWORD[((-136))+r11] |
| 1799 | movaps xmm9,XMMWORD[((-120))+r11] |
| 1800 | movaps xmm10,XMMWORD[((-104))+r11] |
| 1801 | movaps xmm11,XMMWORD[((-88))+r11] |
| 1802 | movaps xmm12,XMMWORD[((-72))+r11] |
| 1803 | movaps xmm13,XMMWORD[((-56))+r11] |
| 1804 | movaps xmm14,XMMWORD[((-40))+r11] |
| 1805 | movaps xmm15,XMMWORD[((-24))+r11] |
| 1806 | lea rsp,[r11] |
| 1807 | |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 1808 | DB 0F3h,0C3h ;repret |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 1809 | |
Robert Sloan | 1c9db53 | 2017-03-13 08:03:59 -0700 | [diff] [blame] | 1810 | $L$SEH_end_rsaz_1024_gather5: |
| 1811 | |
| 1812 | EXTERN OPENSSL_ia32cap_P |
| 1813 | global rsaz_avx2_eligible |
| 1814 | |
| 1815 | ALIGN 32 |
| 1816 | rsaz_avx2_eligible: |
Robert Sloan | 8ff0355 | 2017-06-14 12:40:58 -0700 | [diff] [blame^] | 1817 | lea rax,[OPENSSL_ia32cap_P] |
| 1818 | mov eax,DWORD[8+rax] |
Robert Sloan | 1c9db53 | 2017-03-13 08:03:59 -0700 | [diff] [blame] | 1819 | mov ecx,524544 |
| 1820 | mov edx,0 |
| 1821 | and ecx,eax |
| 1822 | cmp ecx,524544 |
| 1823 | cmove eax,edx |
| 1824 | and eax,32 |
| 1825 | shr eax,5 |
| 1826 | DB 0F3h,0C3h ;repret |
| 1827 | |
| 1828 | |
| 1829 | ALIGN 64 |
| 1830 | $L$and_mask: |
| 1831 | DQ 0x1fffffff,0x1fffffff,0x1fffffff,-1 |
| 1832 | $L$scatter_permd: |
| 1833 | DD 0,2,4,6,7,7,7,7 |
| 1834 | $L$gather_permd: |
| 1835 | DD 0,7,1,7,2,7,3,7 |
| 1836 | $L$inc: |
| 1837 | DD 0,0,0,0,1,1,1,1 |
| 1838 | DD 2,2,2,2,3,3,3,3 |
| 1839 | DD 4,4,4,4,4,4,4,4 |
| 1840 | ALIGN 64 |
| 1841 | EXTERN __imp_RtlVirtualUnwind |
| 1842 | |
| 1843 | ALIGN 16 |
| 1844 | rsaz_se_handler: |
| 1845 | push rsi |
| 1846 | push rdi |
| 1847 | push rbx |
| 1848 | push rbp |
| 1849 | push r12 |
| 1850 | push r13 |
| 1851 | push r14 |
| 1852 | push r15 |
| 1853 | pushfq |
| 1854 | sub rsp,64 |
| 1855 | |
| 1856 | mov rax,QWORD[120+r8] |
| 1857 | mov rbx,QWORD[248+r8] |
| 1858 | |
| 1859 | mov rsi,QWORD[8+r9] |
| 1860 | mov r11,QWORD[56+r9] |
| 1861 | |
| 1862 | mov r10d,DWORD[r11] |
| 1863 | lea r10,[r10*1+rsi] |
| 1864 | cmp rbx,r10 |
| 1865 | jb NEAR $L$common_seh_tail |
| 1866 | |
| 1867 | mov r10d,DWORD[4+r11] |
| 1868 | lea r10,[r10*1+rsi] |
| 1869 | cmp rbx,r10 |
| 1870 | jae NEAR $L$common_seh_tail |
| 1871 | |
| 1872 | mov rbp,QWORD[160+r8] |
| 1873 | |
| 1874 | mov r10d,DWORD[8+r11] |
| 1875 | lea r10,[r10*1+rsi] |
| 1876 | cmp rbx,r10 |
| 1877 | cmovc rax,rbp |
| 1878 | |
| 1879 | mov r15,QWORD[((-48))+rax] |
| 1880 | mov r14,QWORD[((-40))+rax] |
| 1881 | mov r13,QWORD[((-32))+rax] |
| 1882 | mov r12,QWORD[((-24))+rax] |
| 1883 | mov rbp,QWORD[((-16))+rax] |
| 1884 | mov rbx,QWORD[((-8))+rax] |
| 1885 | mov QWORD[240+r8],r15 |
| 1886 | mov QWORD[232+r8],r14 |
| 1887 | mov QWORD[224+r8],r13 |
| 1888 | mov QWORD[216+r8],r12 |
| 1889 | mov QWORD[160+r8],rbp |
| 1890 | mov QWORD[144+r8],rbx |
| 1891 | |
| 1892 | lea rsi,[((-216))+rax] |
| 1893 | lea rdi,[512+r8] |
| 1894 | mov ecx,20 |
| 1895 | DD 0xa548f3fc |
| 1896 | |
| 1897 | $L$common_seh_tail: |
| 1898 | mov rdi,QWORD[8+rax] |
| 1899 | mov rsi,QWORD[16+rax] |
| 1900 | mov QWORD[152+r8],rax |
| 1901 | mov QWORD[168+r8],rsi |
| 1902 | mov QWORD[176+r8],rdi |
| 1903 | |
| 1904 | mov rdi,QWORD[40+r9] |
| 1905 | mov rsi,r8 |
| 1906 | mov ecx,154 |
| 1907 | DD 0xa548f3fc |
| 1908 | |
| 1909 | mov rsi,r9 |
| 1910 | xor rcx,rcx |
| 1911 | mov rdx,QWORD[8+rsi] |
| 1912 | mov r8,QWORD[rsi] |
| 1913 | mov r9,QWORD[16+rsi] |
| 1914 | mov r10,QWORD[40+rsi] |
| 1915 | lea r11,[56+rsi] |
| 1916 | lea r12,[24+rsi] |
| 1917 | mov QWORD[32+rsp],r10 |
| 1918 | mov QWORD[40+rsp],r11 |
| 1919 | mov QWORD[48+rsp],r12 |
| 1920 | mov QWORD[56+rsp],rcx |
| 1921 | call QWORD[__imp_RtlVirtualUnwind] |
| 1922 | |
| 1923 | mov eax,1 |
| 1924 | add rsp,64 |
| 1925 | popfq |
| 1926 | pop r15 |
| 1927 | pop r14 |
| 1928 | pop r13 |
| 1929 | pop r12 |
| 1930 | pop rbp |
| 1931 | pop rbx |
| 1932 | pop rdi |
| 1933 | pop rsi |
| 1934 | DB 0F3h,0C3h ;repret |
| 1935 | |
| 1936 | |
| 1937 | section .pdata rdata align=4 |
| 1938 | ALIGN 4 |
| 1939 | DD $L$SEH_begin_rsaz_1024_sqr_avx2 wrt ..imagebase |
| 1940 | DD $L$SEH_end_rsaz_1024_sqr_avx2 wrt ..imagebase |
| 1941 | DD $L$SEH_info_rsaz_1024_sqr_avx2 wrt ..imagebase |
| 1942 | |
| 1943 | DD $L$SEH_begin_rsaz_1024_mul_avx2 wrt ..imagebase |
| 1944 | DD $L$SEH_end_rsaz_1024_mul_avx2 wrt ..imagebase |
| 1945 | DD $L$SEH_info_rsaz_1024_mul_avx2 wrt ..imagebase |
| 1946 | |
| 1947 | DD $L$SEH_begin_rsaz_1024_gather5 wrt ..imagebase |
| 1948 | DD $L$SEH_end_rsaz_1024_gather5 wrt ..imagebase |
| 1949 | DD $L$SEH_info_rsaz_1024_gather5 wrt ..imagebase |
| 1950 | section .xdata rdata align=8 |
| 1951 | ALIGN 8 |
| 1952 | $L$SEH_info_rsaz_1024_sqr_avx2: |
| 1953 | DB 9,0,0,0 |
| 1954 | DD rsaz_se_handler wrt ..imagebase |
| 1955 | DD $L$sqr_1024_body wrt ..imagebase,$L$sqr_1024_epilogue wrt ..imagebase,$L$sqr_1024_in_tail wrt ..imagebase |
| 1956 | DD 0 |
| 1957 | $L$SEH_info_rsaz_1024_mul_avx2: |
| 1958 | DB 9,0,0,0 |
| 1959 | DD rsaz_se_handler wrt ..imagebase |
| 1960 | DD $L$mul_1024_body wrt ..imagebase,$L$mul_1024_epilogue wrt ..imagebase,$L$mul_1024_in_tail wrt ..imagebase |
| 1961 | DD 0 |
| 1962 | $L$SEH_info_rsaz_1024_gather5: |
| 1963 | DB 0x01,0x36,0x17,0x0b |
| 1964 | DB 0x36,0xf8,0x09,0x00 |
| 1965 | DB 0x31,0xe8,0x08,0x00 |
| 1966 | DB 0x2c,0xd8,0x07,0x00 |
| 1967 | DB 0x27,0xc8,0x06,0x00 |
| 1968 | DB 0x22,0xb8,0x05,0x00 |
| 1969 | DB 0x1d,0xa8,0x04,0x00 |
| 1970 | DB 0x18,0x98,0x03,0x00 |
| 1971 | DB 0x13,0x88,0x02,0x00 |
| 1972 | DB 0x0e,0x78,0x01,0x00 |
| 1973 | DB 0x09,0x68,0x00,0x00 |
| 1974 | DB 0x04,0x01,0x15,0x00 |
| 1975 | DB 0x00,0xb3,0x00,0x00 |