Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / boringssl / fb44824b91c8219bc2dd49195d16c1f07d511a26 / . / win-x86_64 / crypto / bn / x86_64-mont.asm
blob: 1a9da5123060869fe1d3a3339b2fe8c7ca684569 [file] [log] [blame]
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]1default rel
2%define XMMWORD
3%define YMMWORD
4%define ZMMWORD
5section .text code align=64
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]6
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]7
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]8EXTERN OPENSSL_ia32cap_P
9
10global bn_mul_mont
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]11
12ALIGN 16
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]13bn_mul_mont:
14 mov QWORD[8+rsp],rdi ;WIN64 prologue
15 mov QWORD[16+rsp],rsi
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]16 mov rax,rsp
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]17$L$SEH_begin_bn_mul_mont:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]18 mov rdi,rcx
19 mov rsi,rdx
20 mov rdx,r8
21 mov rcx,r9
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]22 mov r8,QWORD[40+rsp]
23 mov r9,QWORD[48+rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]24
25
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]26
27 mov r9d,r9d
28 mov rax,rsp
29
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]30 test r9d,3
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]31 jnz NEAR $L$mul_enter
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]32 cmp r9d,8
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]33 jb NEAR $L$mul_enter
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]34 cmp rdx,rsi
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]35 jne NEAR $L$mul4x_enter
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]36 test r9d,7
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]37 jz NEAR $L$sqr8x_enter
38 jmp NEAR $L$mul4x_enter
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]39
40ALIGN 16
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]41$L$mul_enter:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]42 push rbx
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]43
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]44 push rbp
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]45
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]46 push r12
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]47
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]48 push r13
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]49
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]50 push r14
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]51
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]52 push r15
53
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]54
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]55 neg r9
56 mov r11,rsp
57 lea r10,[((-16))+r9*8+rsp]
58 neg r9
59 and r10,-1024
60
61
62
63
64
65
66
67
68
69 sub r11,r10
70 and r11,-4096
71 lea rsp,[r11*1+r10]
72 mov r11,QWORD[rsp]
73 cmp rsp,r10
74 ja NEAR $L$mul_page_walk
75 jmp NEAR $L$mul_page_walk_done
76
77ALIGN 16
78$L$mul_page_walk:
79 lea rsp,[((-4096))+rsp]
80 mov r11,QWORD[rsp]
81 cmp rsp,r10
82 ja NEAR $L$mul_page_walk
83$L$mul_page_walk_done:
84
85 mov QWORD[8+r9*8+rsp],rax
86
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]87$L$mul_body:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]88 mov r12,rdx
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]89 mov r8,QWORD[r8]
90 mov rbx,QWORD[r12]
91 mov rax,QWORD[rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]92
93 xor r14,r14
94 xor r15,r15
95
96 mov rbp,r8
97 mul rbx
98 mov r10,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]99 mov rax,QWORD[rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]100
101 imul rbp,r10
102 mov r11,rdx
103
104 mul rbp
105 add r10,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]106 mov rax,QWORD[8+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]107 adc rdx,0
108 mov r13,rdx
109
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]110 lea r15,[1+r15]
111 jmp NEAR $L$1st_enter
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]112
113ALIGN 16
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]114$L$1st:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]115 add r13,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]116 mov rax,QWORD[r15*8+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]117 adc rdx,0
118 add r13,r11
119 mov r11,r10
120 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]121 mov QWORD[((-16))+r15*8+rsp],r13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]122 mov r13,rdx
123
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]124$L$1st_enter:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]125 mul rbx
126 add r11,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]127 mov rax,QWORD[r15*8+rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]128 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]129 lea r15,[1+r15]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]130 mov r10,rdx
131
132 mul rbp
133 cmp r15,r9
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]134 jne NEAR $L$1st
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]135
136 add r13,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]137 mov rax,QWORD[rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]138 adc rdx,0
139 add r13,r11
140 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]141 mov QWORD[((-16))+r15*8+rsp],r13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]142 mov r13,rdx
143 mov r11,r10
144
145 xor rdx,rdx
146 add r13,r11
147 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]148 mov QWORD[((-8))+r9*8+rsp],r13
149 mov QWORD[r9*8+rsp],rdx
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]150
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]151 lea r14,[1+r14]
152 jmp NEAR $L$outer
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]153ALIGN 16
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]154$L$outer:
155 mov rbx,QWORD[r14*8+r12]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]156 xor r15,r15
157 mov rbp,r8
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]158 mov r10,QWORD[rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]159 mul rbx
160 add r10,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]161 mov rax,QWORD[rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]162 adc rdx,0
163
164 imul rbp,r10
165 mov r11,rdx
166
167 mul rbp
168 add r10,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]169 mov rax,QWORD[8+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]170 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]171 mov r10,QWORD[8+rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]172 mov r13,rdx
173
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]174 lea r15,[1+r15]
175 jmp NEAR $L$inner_enter
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]176
177ALIGN 16
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]178$L$inner:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]179 add r13,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]180 mov rax,QWORD[r15*8+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]181 adc rdx,0
182 add r13,r10
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]183 mov r10,QWORD[r15*8+rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]184 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]185 mov QWORD[((-16))+r15*8+rsp],r13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]186 mov r13,rdx
187
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]188$L$inner_enter:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]189 mul rbx
190 add r11,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]191 mov rax,QWORD[r15*8+rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]192 adc rdx,0
193 add r10,r11
194 mov r11,rdx
195 adc r11,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]196 lea r15,[1+r15]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]197
198 mul rbp
199 cmp r15,r9
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]200 jne NEAR $L$inner
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]201
202 add r13,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]203 mov rax,QWORD[rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]204 adc rdx,0
205 add r13,r10
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]206 mov r10,QWORD[r15*8+rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]207 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]208 mov QWORD[((-16))+r15*8+rsp],r13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]209 mov r13,rdx
210
211 xor rdx,rdx
212 add r13,r11
213 adc rdx,0
214 add r13,r10
215 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]216 mov QWORD[((-8))+r9*8+rsp],r13
217 mov QWORD[r9*8+rsp],rdx
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]218
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]219 lea r14,[1+r14]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]220 cmp r14,r9
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]221 jb NEAR $L$outer
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]222
223 xor r14,r14
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]224 mov rax,QWORD[rsp]
225 lea rsi,[rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]226 mov r15,r9
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]227 jmp NEAR $L$sub
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]228ALIGN 16
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]229$L$sub: sbb rax,QWORD[r14*8+rcx]
230 mov QWORD[r14*8+rdi],rax
231 mov rax,QWORD[8+r14*8+rsi]
232 lea r14,[1+r14]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]233 dec r15
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]234 jnz NEAR $L$sub
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]235
236 sbb rax,0
237 xor r14,r14
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]238 and rsi,rax
239 not rax
240 mov rcx,rdi
241 and rcx,rax
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]242 mov r15,r9
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]243 or rsi,rcx
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]244ALIGN 16
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]245$L$copy:
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]246 mov rax,QWORD[r14*8+rsi]
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]247 mov QWORD[r14*8+rsp],r14
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]248 mov QWORD[r14*8+rdi],rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]249 lea r14,[1+r14]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]250 sub r15,1
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]251 jnz NEAR $L$copy
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]252
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]253 mov rsi,QWORD[8+r9*8+rsp]
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]254
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]255 mov rax,1
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]256 mov r15,QWORD[((-48))+rsi]
257
258 mov r14,QWORD[((-40))+rsi]
259
260 mov r13,QWORD[((-32))+rsi]
261
262 mov r12,QWORD[((-24))+rsi]
263
264 mov rbp,QWORD[((-16))+rsi]
265
266 mov rbx,QWORD[((-8))+rsi]
267
268 lea rsp,[rsi]
269
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]270$L$mul_epilogue:
271 mov rdi,QWORD[8+rsp] ;WIN64 epilogue
272 mov rsi,QWORD[16+rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]273 DB 0F3h,0C3h ;repret
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]274
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]275$L$SEH_end_bn_mul_mont:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]276
277ALIGN 16
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]278bn_mul4x_mont:
279 mov QWORD[8+rsp],rdi ;WIN64 prologue
280 mov QWORD[16+rsp],rsi
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]281 mov rax,rsp
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]282$L$SEH_begin_bn_mul4x_mont:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]283 mov rdi,rcx
284 mov rsi,rdx
285 mov rdx,r8
286 mov rcx,r9
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]287 mov r8,QWORD[40+rsp]
288 mov r9,QWORD[48+rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]289
290
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]291
292 mov r9d,r9d
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]293 mov rax,rsp
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]294
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]295$L$mul4x_enter:
296 push rbx
297
298 push rbp
299
300 push r12
301
302 push r13
303
304 push r14
305
306 push r15
307
308
309 neg r9
310 mov r11,rsp
311 lea r10,[((-32))+r9*8+rsp]
312 neg r9
313 and r10,-1024
314
315 sub r11,r10
316 and r11,-4096
317 lea rsp,[r11*1+r10]
318 mov r11,QWORD[rsp]
319 cmp rsp,r10
320 ja NEAR $L$mul4x_page_walk
321 jmp NEAR $L$mul4x_page_walk_done
322
323$L$mul4x_page_walk:
324 lea rsp,[((-4096))+rsp]
325 mov r11,QWORD[rsp]
326 cmp rsp,r10
327 ja NEAR $L$mul4x_page_walk
328$L$mul4x_page_walk_done:
329
330 mov QWORD[8+r9*8+rsp],rax
331
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]332$L$mul4x_body:
333 mov QWORD[16+r9*8+rsp],rdi
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]334 mov r12,rdx
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]335 mov r8,QWORD[r8]
336 mov rbx,QWORD[r12]
337 mov rax,QWORD[rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]338
339 xor r14,r14
340 xor r15,r15
341
342 mov rbp,r8
343 mul rbx
344 mov r10,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]345 mov rax,QWORD[rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]346
347 imul rbp,r10
348 mov r11,rdx
349
350 mul rbp
351 add r10,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]352 mov rax,QWORD[8+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]353 adc rdx,0
354 mov rdi,rdx
355
356 mul rbx
357 add r11,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]358 mov rax,QWORD[8+rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]359 adc rdx,0
360 mov r10,rdx
361
362 mul rbp
363 add rdi,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]364 mov rax,QWORD[16+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]365 adc rdx,0
366 add rdi,r11
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]367 lea r15,[4+r15]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]368 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]369 mov QWORD[rsp],rdi
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]370 mov r13,rdx
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]371 jmp NEAR $L$1st4x
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]372ALIGN 16
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]373$L$1st4x:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]374 mul rbx
375 add r10,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]376 mov rax,QWORD[((-16))+r15*8+rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]377 adc rdx,0
378 mov r11,rdx
379
380 mul rbp
381 add r13,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]382 mov rax,QWORD[((-8))+r15*8+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]383 adc rdx,0
384 add r13,r10
385 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]386 mov QWORD[((-24))+r15*8+rsp],r13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]387 mov rdi,rdx
388
389 mul rbx
390 add r11,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]391 mov rax,QWORD[((-8))+r15*8+rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]392 adc rdx,0
393 mov r10,rdx
394
395 mul rbp
396 add rdi,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]397 mov rax,QWORD[r15*8+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]398 adc rdx,0
399 add rdi,r11
400 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]401 mov QWORD[((-16))+r15*8+rsp],rdi
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]402 mov r13,rdx
403
404 mul rbx
405 add r10,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]406 mov rax,QWORD[r15*8+rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]407 adc rdx,0
408 mov r11,rdx
409
410 mul rbp
411 add r13,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]412 mov rax,QWORD[8+r15*8+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]413 adc rdx,0
414 add r13,r10
415 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]416 mov QWORD[((-8))+r15*8+rsp],r13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]417 mov rdi,rdx
418
419 mul rbx
420 add r11,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]421 mov rax,QWORD[8+r15*8+rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]422 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]423 lea r15,[4+r15]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]424 mov r10,rdx
425
426 mul rbp
427 add rdi,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]428 mov rax,QWORD[((-16))+r15*8+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]429 adc rdx,0
430 add rdi,r11
431 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]432 mov QWORD[((-32))+r15*8+rsp],rdi
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]433 mov r13,rdx
434 cmp r15,r9
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]435 jb NEAR $L$1st4x
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]436
437 mul rbx
438 add r10,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]439 mov rax,QWORD[((-16))+r15*8+rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]440 adc rdx,0
441 mov r11,rdx
442
443 mul rbp
444 add r13,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]445 mov rax,QWORD[((-8))+r15*8+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]446 adc rdx,0
447 add r13,r10
448 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]449 mov QWORD[((-24))+r15*8+rsp],r13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]450 mov rdi,rdx
451
452 mul rbx
453 add r11,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]454 mov rax,QWORD[((-8))+r15*8+rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]455 adc rdx,0
456 mov r10,rdx
457
458 mul rbp
459 add rdi,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]460 mov rax,QWORD[rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]461 adc rdx,0
462 add rdi,r11
463 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]464 mov QWORD[((-16))+r15*8+rsp],rdi
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]465 mov r13,rdx
466
467 xor rdi,rdi
468 add r13,r10
469 adc rdi,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]470 mov QWORD[((-8))+r15*8+rsp],r13
471 mov QWORD[r15*8+rsp],rdi
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]472
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]473 lea r14,[1+r14]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]474ALIGN 4
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]475$L$outer4x:
476 mov rbx,QWORD[r14*8+r12]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]477 xor r15,r15
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]478 mov r10,QWORD[rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]479 mov rbp,r8
480 mul rbx
481 add r10,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]482 mov rax,QWORD[rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]483 adc rdx,0
484
485 imul rbp,r10
486 mov r11,rdx
487
488 mul rbp
489 add r10,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]490 mov rax,QWORD[8+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]491 adc rdx,0
492 mov rdi,rdx
493
494 mul rbx
495 add r11,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]496 mov rax,QWORD[8+rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]497 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]498 add r11,QWORD[8+rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]499 adc rdx,0
500 mov r10,rdx
501
502 mul rbp
503 add rdi,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]504 mov rax,QWORD[16+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]505 adc rdx,0
506 add rdi,r11
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]507 lea r15,[4+r15]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]508 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]509 mov QWORD[rsp],rdi
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]510 mov r13,rdx
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]511 jmp NEAR $L$inner4x
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]512ALIGN 16
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]513$L$inner4x:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]514 mul rbx
515 add r10,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]516 mov rax,QWORD[((-16))+r15*8+rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]517 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]518 add r10,QWORD[((-16))+r15*8+rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]519 adc rdx,0
520 mov r11,rdx
521
522 mul rbp
523 add r13,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]524 mov rax,QWORD[((-8))+r15*8+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]525 adc rdx,0
526 add r13,r10
527 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]528 mov QWORD[((-24))+r15*8+rsp],r13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]529 mov rdi,rdx
530
531 mul rbx
532 add r11,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]533 mov rax,QWORD[((-8))+r15*8+rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]534 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]535 add r11,QWORD[((-8))+r15*8+rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]536 adc rdx,0
537 mov r10,rdx
538
539 mul rbp
540 add rdi,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]541 mov rax,QWORD[r15*8+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]542 adc rdx,0
543 add rdi,r11
544 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]545 mov QWORD[((-16))+r15*8+rsp],rdi
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]546 mov r13,rdx
547
548 mul rbx
549 add r10,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]550 mov rax,QWORD[r15*8+rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]551 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]552 add r10,QWORD[r15*8+rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]553 adc rdx,0
554 mov r11,rdx
555
556 mul rbp
557 add r13,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]558 mov rax,QWORD[8+r15*8+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]559 adc rdx,0
560 add r13,r10
561 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]562 mov QWORD[((-8))+r15*8+rsp],r13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]563 mov rdi,rdx
564
565 mul rbx
566 add r11,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]567 mov rax,QWORD[8+r15*8+rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]568 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]569 add r11,QWORD[8+r15*8+rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]570 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]571 lea r15,[4+r15]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]572 mov r10,rdx
573
574 mul rbp
575 add rdi,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]576 mov rax,QWORD[((-16))+r15*8+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]577 adc rdx,0
578 add rdi,r11
579 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]580 mov QWORD[((-32))+r15*8+rsp],rdi
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]581 mov r13,rdx
582 cmp r15,r9
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]583 jb NEAR $L$inner4x
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]584
585 mul rbx
586 add r10,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]587 mov rax,QWORD[((-16))+r15*8+rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]588 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]589 add r10,QWORD[((-16))+r15*8+rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]590 adc rdx,0
591 mov r11,rdx
592
593 mul rbp
594 add r13,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]595 mov rax,QWORD[((-8))+r15*8+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]596 adc rdx,0
597 add r13,r10
598 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]599 mov QWORD[((-24))+r15*8+rsp],r13
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]600 mov rdi,rdx
601
602 mul rbx
603 add r11,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]604 mov rax,QWORD[((-8))+r15*8+rcx]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]605 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]606 add r11,QWORD[((-8))+r15*8+rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]607 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]608 lea r14,[1+r14]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]609 mov r10,rdx
610
611 mul rbp
612 add rdi,rax
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]613 mov rax,QWORD[rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]614 adc rdx,0
615 add rdi,r11
616 adc rdx,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]617 mov QWORD[((-16))+r15*8+rsp],rdi
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]618 mov r13,rdx
619
620 xor rdi,rdi
621 add r13,r10
622 adc rdi,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]623 add r13,QWORD[r9*8+rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]624 adc rdi,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]625 mov QWORD[((-8))+r15*8+rsp],r13
626 mov QWORD[r15*8+rsp],rdi
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]627
628 cmp r14,r9
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]629 jb NEAR $L$outer4x
630 mov rdi,QWORD[16+r9*8+rsp]
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]631 lea r15,[((-4))+r9]
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]632 mov rax,QWORD[rsp]
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]633 pxor xmm0,xmm0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]634 mov rdx,QWORD[8+rsp]
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]635 shr r15,2
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]636 lea rsi,[rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]637 xor r14,r14
638
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]639 sub rax,QWORD[rcx]
640 mov rbx,QWORD[16+rsi]
641 mov rbp,QWORD[24+rsi]
642 sbb rdx,QWORD[8+rcx]
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]643 jmp NEAR $L$sub4x
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]644ALIGN 16
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]645$L$sub4x:
646 mov QWORD[r14*8+rdi],rax
647 mov QWORD[8+r14*8+rdi],rdx
648 sbb rbx,QWORD[16+r14*8+rcx]
649 mov rax,QWORD[32+r14*8+rsi]
650 mov rdx,QWORD[40+r14*8+rsi]
651 sbb rbp,QWORD[24+r14*8+rcx]
652 mov QWORD[16+r14*8+rdi],rbx
653 mov QWORD[24+r14*8+rdi],rbp
654 sbb rax,QWORD[32+r14*8+rcx]
655 mov rbx,QWORD[48+r14*8+rsi]
656 mov rbp,QWORD[56+r14*8+rsi]
657 sbb rdx,QWORD[40+r14*8+rcx]
658 lea r14,[4+r14]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]659 dec r15
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]660 jnz NEAR $L$sub4x
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]661
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]662 mov QWORD[r14*8+rdi],rax
663 mov rax,QWORD[32+r14*8+rsi]
664 sbb rbx,QWORD[16+r14*8+rcx]
665 mov QWORD[8+r14*8+rdi],rdx
666 sbb rbp,QWORD[24+r14*8+rcx]
667 mov QWORD[16+r14*8+rdi],rbx
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]668
669 sbb rax,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]670 mov QWORD[24+r14*8+rdi],rbp
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]671 xor r14,r14
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]672 and rsi,rax
673 not rax
674 mov rcx,rdi
675 and rcx,rax
676 lea r15,[((-4))+r9]
677 or rsi,rcx
678 shr r15,2
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]679
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]680 movdqu xmm1,XMMWORD[rsi]
681 movdqa XMMWORD[rsp],xmm0
682 movdqu XMMWORD[rdi],xmm1
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]683 jmp NEAR $L$copy4x
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]684ALIGN 16
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]685$L$copy4x:
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]686 movdqu xmm2,XMMWORD[16+r14*1+rsi]
687 movdqu xmm1,XMMWORD[32+r14*1+rsi]
688 movdqa XMMWORD[16+r14*1+rsp],xmm0
689 movdqu XMMWORD[16+r14*1+rdi],xmm2
690 movdqa XMMWORD[32+r14*1+rsp],xmm0
691 movdqu XMMWORD[32+r14*1+rdi],xmm1
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]692 lea r14,[32+r14]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]693 dec r15
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]694 jnz NEAR $L$copy4x
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]695
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]696 movdqu xmm2,XMMWORD[16+r14*1+rsi]
697 movdqa XMMWORD[16+r14*1+rsp],xmm0
698 movdqu XMMWORD[16+r14*1+rdi],xmm2
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]699 mov rsi,QWORD[8+r9*8+rsp]
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]700
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]701 mov rax,1
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]702 mov r15,QWORD[((-48))+rsi]
703
704 mov r14,QWORD[((-40))+rsi]
705
706 mov r13,QWORD[((-32))+rsi]
707
708 mov r12,QWORD[((-24))+rsi]
709
710 mov rbp,QWORD[((-16))+rsi]
711
712 mov rbx,QWORD[((-8))+rsi]
713
714 lea rsp,[rsi]
715
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]716$L$mul4x_epilogue:
717 mov rdi,QWORD[8+rsp] ;WIN64 epilogue
718 mov rsi,QWORD[16+rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]719 DB 0F3h,0C3h ;repret
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]720
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]721$L$SEH_end_bn_mul4x_mont:
722EXTERN bn_sqr8x_internal
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]723
724
725ALIGN 32
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]726bn_sqr8x_mont:
727 mov QWORD[8+rsp],rdi ;WIN64 prologue
728 mov QWORD[16+rsp],rsi
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]729 mov rax,rsp
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]730$L$SEH_begin_bn_sqr8x_mont:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]731 mov rdi,rcx
732 mov rsi,rdx
733 mov rdx,r8
734 mov rcx,r9
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]735 mov r8,QWORD[40+rsp]
736 mov r9,QWORD[48+rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]737
738
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]739
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]740 mov rax,rsp
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]741
742$L$sqr8x_enter:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]743 push rbx
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]744
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]745 push rbp
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]746
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]747 push r12
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]748
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]749 push r13
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]750
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]751 push r14
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]752
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]753 push r15
754
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]755$L$sqr8x_prologue:
756
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]757 mov r10d,r9d
758 shl r9d,3
759 shl r10,3+2
760 neg r9
761
762
763
764
765
766
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]767 lea r11,[((-64))+r9*2+rsp]
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]768 mov rbp,rsp
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]769 mov r8,QWORD[r8]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]770 sub r11,rsi
771 and r11,4095
772 cmp r10,r11
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]773 jb NEAR $L$sqr8x_sp_alt
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]774 sub rbp,r11
775 lea rbp,[((-64))+r9*2+rbp]
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]776 jmp NEAR $L$sqr8x_sp_done
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]777
778ALIGN 32
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]779$L$sqr8x_sp_alt:
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]780 lea r10,[((4096-64))+r9*2]
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]781 lea rbp,[((-64))+r9*2+rbp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]782 sub r11,r10
783 mov r10,0
784 cmovc r11,r10
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]785 sub rbp,r11
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]786$L$sqr8x_sp_done:
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]787 and rbp,-64
788 mov r11,rsp
789 sub r11,rbp
790 and r11,-4096
791 lea rsp,[rbp*1+r11]
792 mov r10,QWORD[rsp]
793 cmp rsp,rbp
794 ja NEAR $L$sqr8x_page_walk
795 jmp NEAR $L$sqr8x_page_walk_done
796
797ALIGN 16
798$L$sqr8x_page_walk:
799 lea rsp,[((-4096))+rsp]
800 mov r10,QWORD[rsp]
801 cmp rsp,rbp
802 ja NEAR $L$sqr8x_page_walk
803$L$sqr8x_page_walk_done:
804
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]805 mov r10,r9
806 neg r9
807
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]808 mov QWORD[32+rsp],r8
809 mov QWORD[40+rsp],rax
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]810
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]811$L$sqr8x_body:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]812
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]813DB 102,72,15,110,209
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]814 pxor xmm0,xmm0
815DB 102,72,15,110,207
816DB 102,73,15,110,218
817 call bn_sqr8x_internal
818
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]819
820
821
822 lea rbx,[r9*1+rdi]
823 mov rcx,r9
824 mov rdx,r9
825DB 102,72,15,126,207
826 sar rcx,3+2
827 jmp NEAR $L$sqr8x_sub
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]828
829ALIGN 32
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]830$L$sqr8x_sub:
831 mov r12,QWORD[rbx]
832 mov r13,QWORD[8+rbx]
833 mov r14,QWORD[16+rbx]
834 mov r15,QWORD[24+rbx]
835 lea rbx,[32+rbx]
836 sbb r12,QWORD[rbp]
837 sbb r13,QWORD[8+rbp]
838 sbb r14,QWORD[16+rbp]
839 sbb r15,QWORD[24+rbp]
840 lea rbp,[32+rbp]
841 mov QWORD[rdi],r12
842 mov QWORD[8+rdi],r13
843 mov QWORD[16+rdi],r14
844 mov QWORD[24+rdi],r15
845 lea rdi,[32+rdi]
846 inc rcx
847 jnz NEAR $L$sqr8x_sub
848
849 sbb rax,0
850 lea rbx,[r9*1+rbx]
851 lea rdi,[r9*1+rdi]
852
853DB 102,72,15,110,200
854 pxor xmm0,xmm0
855 pshufd xmm1,xmm1,0
856 mov rsi,QWORD[40+rsp]
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]857
David Benjamin4969cc92016-04-22 15:02:23 -0400[diff] [blame]858 jmp NEAR $L$sqr8x_cond_copy
859
860ALIGN 32
861$L$sqr8x_cond_copy:
862 movdqa xmm2,XMMWORD[rbx]
863 movdqa xmm3,XMMWORD[16+rbx]
864 lea rbx,[32+rbx]
865 movdqu xmm4,XMMWORD[rdi]
866 movdqu xmm5,XMMWORD[16+rdi]
867 lea rdi,[32+rdi]
868 movdqa XMMWORD[(-32)+rbx],xmm0
869 movdqa XMMWORD[(-16)+rbx],xmm0
870 movdqa XMMWORD[(-32)+rdx*1+rbx],xmm0
871 movdqa XMMWORD[(-16)+rdx*1+rbx],xmm0
872 pcmpeqd xmm0,xmm1
873 pand xmm2,xmm1
874 pand xmm3,xmm1
875 pand xmm4,xmm0
876 pand xmm5,xmm0
877 pxor xmm0,xmm0
878 por xmm4,xmm2
879 por xmm5,xmm3
880 movdqu XMMWORD[(-32)+rdi],xmm4
881 movdqu XMMWORD[(-16)+rdi],xmm5
882 add r9,32
883 jnz NEAR $L$sqr8x_cond_copy
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]884
885 mov rax,1
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]886 mov r15,QWORD[((-48))+rsi]
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]887
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]888 mov r14,QWORD[((-40))+rsi]
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]889
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]890 mov r13,QWORD[((-32))+rsi]
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]891
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]892 mov r12,QWORD[((-24))+rsi]
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]893
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]894 mov rbp,QWORD[((-16))+rsi]
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]895
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]896 mov rbx,QWORD[((-8))+rsi]
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]897
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]898 lea rsp,[rsi]
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]899
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]900$L$sqr8x_epilogue:
901 mov rdi,QWORD[8+rsp] ;WIN64 epilogue
902 mov rsi,QWORD[16+rsp]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]903 DB 0F3h,0C3h ;repret
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]904
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]905$L$SEH_end_bn_sqr8x_mont:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]906DB 77,111,110,116,103,111,109,101,114,121,32,77,117,108,116,105
907DB 112,108,105,99,97,116,105,111,110,32,102,111,114,32,120,56
908DB 54,95,54,52,44,32,67,82,89,80,84,79,71,65,77,83
909DB 32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115
910DB 115,108,46,111,114,103,62,0
911ALIGN 16
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]912EXTERN __imp_RtlVirtualUnwind
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]913
914ALIGN 16
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]915mul_handler:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]916 push rsi
917 push rdi
918 push rbx
919 push rbp
920 push r12
921 push r13
922 push r14
923 push r15
924 pushfq
925 sub rsp,64
926
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]927 mov rax,QWORD[120+r8]
928 mov rbx,QWORD[248+r8]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]929
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]930 mov rsi,QWORD[8+r9]
931 mov r11,QWORD[56+r9]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]932
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]933 mov r10d,DWORD[r11]
934 lea r10,[r10*1+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]935 cmp rbx,r10
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]936 jb NEAR $L$common_seh_tail
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]937
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]938 mov rax,QWORD[152+r8]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]939
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]940 mov r10d,DWORD[4+r11]
941 lea r10,[r10*1+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]942 cmp rbx,r10
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]943 jae NEAR $L$common_seh_tail
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]944
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]945 mov r10,QWORD[192+r8]
946 mov rax,QWORD[8+r10*8+rax]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]947
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]948 jmp NEAR $L$common_pop_regs
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]949
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]950
951
952ALIGN 16
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]953sqr_handler:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]954 push rsi
955 push rdi
956 push rbx
957 push rbp
958 push r12
959 push r13
960 push r14
961 push r15
962 pushfq
963 sub rsp,64
964
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]965 mov rax,QWORD[120+r8]
966 mov rbx,QWORD[248+r8]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]967
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]968 mov rsi,QWORD[8+r9]
969 mov r11,QWORD[56+r9]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]970
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]971 mov r10d,DWORD[r11]
972 lea r10,[r10*1+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]973 cmp rbx,r10
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]974 jb NEAR $L$common_seh_tail
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]975
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]976 mov r10d,DWORD[4+r11]
977 lea r10,[r10*1+rsi]
978 cmp rbx,r10
979 jb NEAR $L$common_pop_regs
980
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]981 mov rax,QWORD[152+r8]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]982
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]983 mov r10d,DWORD[8+r11]
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]984 lea r10,[r10*1+rsi]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]985 cmp rbx,r10
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]986 jae NEAR $L$common_seh_tail
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]987
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]988 mov rax,QWORD[40+rax]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]989
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]990$L$common_pop_regs:
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]991 mov rbx,QWORD[((-8))+rax]
992 mov rbp,QWORD[((-16))+rax]
993 mov r12,QWORD[((-24))+rax]
994 mov r13,QWORD[((-32))+rax]
995 mov r14,QWORD[((-40))+rax]
996 mov r15,QWORD[((-48))+rax]
997 mov QWORD[144+r8],rbx
998 mov QWORD[160+r8],rbp
999 mov QWORD[216+r8],r12
1000 mov QWORD[224+r8],r13
1001 mov QWORD[232+r8],r14
1002 mov QWORD[240+r8],r15
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1003
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]1004$L$common_seh_tail:
1005 mov rdi,QWORD[8+rax]
1006 mov rsi,QWORD[16+rax]
1007 mov QWORD[152+r8],rax
1008 mov QWORD[168+r8],rsi
1009 mov QWORD[176+r8],rdi
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1010
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]1011 mov rdi,QWORD[40+r9]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1012 mov rsi,r8
1013 mov ecx,154
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]1014 DD 0xa548f3fc
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1015
1016 mov rsi,r9
1017 xor rcx,rcx
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]1018 mov rdx,QWORD[8+rsi]
1019 mov r8,QWORD[rsi]
1020 mov r9,QWORD[16+rsi]
1021 mov r10,QWORD[40+rsi]
1022 lea r11,[56+rsi]
1023 lea r12,[24+rsi]
1024 mov QWORD[32+rsp],r10
1025 mov QWORD[40+rsp],r11
1026 mov QWORD[48+rsp],r12
1027 mov QWORD[56+rsp],rcx
1028 call QWORD[__imp_RtlVirtualUnwind]
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1029
1030 mov eax,1
1031 add rsp,64
1032 popfq
1033 pop r15
1034 pop r14
1035 pop r13
1036 pop r12
1037 pop rbp
1038 pop rbx
1039 pop rdi
1040 pop rsi
1041 DB 0F3h,0C3h ;repret
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1042
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]1043
1044section .pdata rdata align=4
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1045ALIGN 4
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]1046 DD $L$SEH_begin_bn_mul_mont wrt ..imagebase
1047 DD $L$SEH_end_bn_mul_mont wrt ..imagebase
1048 DD $L$SEH_info_bn_mul_mont wrt ..imagebase
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1049
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]1050 DD $L$SEH_begin_bn_mul4x_mont wrt ..imagebase
1051 DD $L$SEH_end_bn_mul4x_mont wrt ..imagebase
1052 DD $L$SEH_info_bn_mul4x_mont wrt ..imagebase
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1053
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]1054 DD $L$SEH_begin_bn_sqr8x_mont wrt ..imagebase
1055 DD $L$SEH_end_bn_sqr8x_mont wrt ..imagebase
1056 DD $L$SEH_info_bn_sqr8x_mont wrt ..imagebase
1057section .xdata rdata align=8
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1058ALIGN 8
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]1059$L$SEH_info_bn_mul_mont:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1060DB 9,0,0,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]1061 DD mul_handler wrt ..imagebase
1062 DD $L$mul_body wrt ..imagebase,$L$mul_epilogue wrt ..imagebase
1063$L$SEH_info_bn_mul4x_mont:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1064DB 9,0,0,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]1065 DD mul_handler wrt ..imagebase
1066 DD $L$mul4x_body wrt ..imagebase,$L$mul4x_epilogue wrt ..imagebase
1067$L$SEH_info_bn_sqr8x_mont:
Adam Langleyd9e397b2015-01-22 14:27:53 -0800[diff] [blame]1068DB 9,0,0,0
Adam Langleye9ada862015-05-11 17:20:37 -0700[diff] [blame]1069 DD sqr_handler wrt ..imagebase
Robert Sloana94fe052017-02-21 08:49:28 -0800[diff] [blame]1070 DD $L$sqr8x_prologue wrt ..imagebase,$L$sqr8x_body wrt ..imagebase,$L$sqr8x_epilogue wrt ..imagebase
1071ALIGN 8