Blame - win-x86_64/crypto/fipsmodule/x86_64-mont5.asm - platform/external/boringssl

blob: 1bcbc5d097a227a3b2895662e5a0ef24b7a0e834 [file] [log] [blame]

Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1	default rel
				2	%define XMMWORD
				3	%define YMMWORD
				4	%define ZMMWORD
				5	section .text code align=64
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	8	EXTERN OPENSSL_ia32cap_P
				9
				10	global bn_mul_mont_gather5
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	11
				12	ALIGN 64
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	13	bn_mul_mont_gather5:
				14	mov QWORD[8+rsp],rdi ;WIN64 prologue
				15	mov QWORD[16+rsp],rsi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	16	mov rax,rsp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	17	$L$SEH_begin_bn_mul_mont_gather5:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	18	mov rdi,rcx
				19	mov rsi,rdx
				20	mov rdx,r8
				21	mov rcx,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	22	mov r8,QWORD[40+rsp]
				23	mov r9,QWORD[48+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	24
				25
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	26
				27	mov r9d,r9d
				28	mov rax,rsp
				29
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	30	test r9d,7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	31	jnz NEAR $L$mul_enter
				32	jmp NEAR $L$mul4x_enter
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	33
				34	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	35	$L$mul_enter:
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	36	movd xmm5,DWORD[56+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	37	push rbx
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	38
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	39	push rbp
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	40
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	41	push r12
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	42
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	43	push r13
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	44
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	45	push r14
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	46
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	47	push r15
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	48
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	49
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	50	neg r9
				51	mov r11,rsp
				52	lea r10,[((-280))+r9*8+rsp]
				53	neg r9
				54	and r10,-1024
				55
				56
				57
				58
				59
				60
				61
				62
				63
				64	sub r11,r10
				65	and r11,-4096
				66	lea rsp,[r11*1+r10]
				67	mov r11,QWORD[rsp]
				68	cmp rsp,r10
				69	ja NEAR $L$mul_page_walk
				70	jmp NEAR $L$mul_page_walk_done
				71
				72	$L$mul_page_walk:
				73	lea rsp,[((-4096))+rsp]
				74	mov r11,QWORD[rsp]
				75	cmp rsp,r10
				76	ja NEAR $L$mul_page_walk
				77	$L$mul_page_walk_done:
				78
				79	lea r10,[$L$inc]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	80	mov QWORD[8+r9*8+rsp],rax
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	81
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	82	$L$mul_body:
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	83
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	84	lea r12,[128+rdx]
				85	movdqa xmm0,XMMWORD[r10]
				86	movdqa xmm1,XMMWORD[16+r10]
				87	lea r10,[((24-112))+r9*8+rsp]
				88	and r10,-16
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	89
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	90	pshufd xmm5,xmm5,0
				91	movdqa xmm4,xmm1
				92	movdqa xmm2,xmm1
				93	paddd xmm1,xmm0
				94	pcmpeqd xmm0,xmm5
				95	DB 0x67
				96	movdqa xmm3,xmm4
				97	paddd xmm2,xmm1
				98	pcmpeqd xmm1,xmm5
				99	movdqa XMMWORD[112+r10],xmm0
				100	movdqa xmm0,xmm4
				101
				102	paddd xmm3,xmm2
				103	pcmpeqd xmm2,xmm5
				104	movdqa XMMWORD[128+r10],xmm1
				105	movdqa xmm1,xmm4
				106
				107	paddd xmm0,xmm3
				108	pcmpeqd xmm3,xmm5
				109	movdqa XMMWORD[144+r10],xmm2
				110	movdqa xmm2,xmm4
				111
				112	paddd xmm1,xmm0
				113	pcmpeqd xmm0,xmm5
				114	movdqa XMMWORD[160+r10],xmm3
				115	movdqa xmm3,xmm4
				116	paddd xmm2,xmm1
				117	pcmpeqd xmm1,xmm5
				118	movdqa XMMWORD[176+r10],xmm0
				119	movdqa xmm0,xmm4
				120
				121	paddd xmm3,xmm2
				122	pcmpeqd xmm2,xmm5
				123	movdqa XMMWORD[192+r10],xmm1
				124	movdqa xmm1,xmm4
				125
				126	paddd xmm0,xmm3
				127	pcmpeqd xmm3,xmm5
				128	movdqa XMMWORD[208+r10],xmm2
				129	movdqa xmm2,xmm4
				130
				131	paddd xmm1,xmm0
				132	pcmpeqd xmm0,xmm5
				133	movdqa XMMWORD[224+r10],xmm3
				134	movdqa xmm3,xmm4
				135	paddd xmm2,xmm1
				136	pcmpeqd xmm1,xmm5
				137	movdqa XMMWORD[240+r10],xmm0
				138	movdqa xmm0,xmm4
				139
				140	paddd xmm3,xmm2
				141	pcmpeqd xmm2,xmm5
				142	movdqa XMMWORD[256+r10],xmm1
				143	movdqa xmm1,xmm4
				144
				145	paddd xmm0,xmm3
				146	pcmpeqd xmm3,xmm5
				147	movdqa XMMWORD[272+r10],xmm2
				148	movdqa xmm2,xmm4
				149
				150	paddd xmm1,xmm0
				151	pcmpeqd xmm0,xmm5
				152	movdqa XMMWORD[288+r10],xmm3
				153	movdqa xmm3,xmm4
				154	paddd xmm2,xmm1
				155	pcmpeqd xmm1,xmm5
				156	movdqa XMMWORD[304+r10],xmm0
				157
				158	paddd xmm3,xmm2
				159	DB 0x67
				160	pcmpeqd xmm2,xmm5
				161	movdqa XMMWORD[320+r10],xmm1
				162
				163	pcmpeqd xmm3,xmm5
				164	movdqa XMMWORD[336+r10],xmm2
				165	pand xmm0,XMMWORD[64+r12]
				166
				167	pand xmm1,XMMWORD[80+r12]
				168	pand xmm2,XMMWORD[96+r12]
				169	movdqa XMMWORD[352+r10],xmm3
				170	pand xmm3,XMMWORD[112+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	171	por xmm0,xmm2
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	172	por xmm1,xmm3
				173	movdqa xmm4,XMMWORD[((-128))+r12]
				174	movdqa xmm5,XMMWORD[((-112))+r12]
				175	movdqa xmm2,XMMWORD[((-96))+r12]
				176	pand xmm4,XMMWORD[112+r10]
				177	movdqa xmm3,XMMWORD[((-80))+r12]
				178	pand xmm5,XMMWORD[128+r10]
				179	por xmm0,xmm4
				180	pand xmm2,XMMWORD[144+r10]
				181	por xmm1,xmm5
				182	pand xmm3,XMMWORD[160+r10]
				183	por xmm0,xmm2
				184	por xmm1,xmm3
				185	movdqa xmm4,XMMWORD[((-64))+r12]
				186	movdqa xmm5,XMMWORD[((-48))+r12]
				187	movdqa xmm2,XMMWORD[((-32))+r12]
				188	pand xmm4,XMMWORD[176+r10]
				189	movdqa xmm3,XMMWORD[((-16))+r12]
				190	pand xmm5,XMMWORD[192+r10]
				191	por xmm0,xmm4
				192	pand xmm2,XMMWORD[208+r10]
				193	por xmm1,xmm5
				194	pand xmm3,XMMWORD[224+r10]
				195	por xmm0,xmm2
				196	por xmm1,xmm3
				197	movdqa xmm4,XMMWORD[r12]
				198	movdqa xmm5,XMMWORD[16+r12]
				199	movdqa xmm2,XMMWORD[32+r12]
				200	pand xmm4,XMMWORD[240+r10]
				201	movdqa xmm3,XMMWORD[48+r12]
				202	pand xmm5,XMMWORD[256+r10]
				203	por xmm0,xmm4
				204	pand xmm2,XMMWORD[272+r10]
				205	por xmm1,xmm5
				206	pand xmm3,XMMWORD[288+r10]
				207	por xmm0,xmm2
				208	por xmm1,xmm3
				209	por xmm0,xmm1
				210	pshufd xmm1,xmm0,0x4e
				211	por xmm0,xmm1
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	212	lea r12,[256+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	213	DB 102,72,15,126,195
				214
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	215	mov r8,QWORD[r8]
				216	mov rax,QWORD[rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	217
				218	xor r14,r14
				219	xor r15,r15
				220
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	221	mov rbp,r8
				222	mul rbx
				223	mov r10,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	224	mov rax,QWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	225
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	226	imul rbp,r10
				227	mov r11,rdx
				228
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	229	mul rbp
				230	add r10,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	231	mov rax,QWORD[8+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	232	adc rdx,0
				233	mov r13,rdx
				234
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	235	lea r15,[1+r15]
				236	jmp NEAR $L$1st_enter
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	237
				238	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	239	$L$1st:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	240	add r13,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	241	mov rax,QWORD[r15*8+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	242	adc rdx,0
				243	add r13,r11
				244	mov r11,r10
				245	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	246	mov QWORD[((-16))+r15*8+rsp],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	247	mov r13,rdx
				248
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	249	$L$1st_enter:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	250	mul rbx
				251	add r11,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	252	mov rax,QWORD[r15*8+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	253	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	254	lea r15,[1+r15]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	255	mov r10,rdx
				256
				257	mul rbp
				258	cmp r15,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	259	jne NEAR $L$1st
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	260
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	261
				262	add r13,rax
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	263	adc rdx,0
				264	add r13,r11
				265	adc rdx,0
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	266	mov QWORD[((-16))+r9*8+rsp],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	267	mov r13,rdx
				268	mov r11,r10
				269
				270	xor rdx,rdx
				271	add r13,r11
				272	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	273	mov QWORD[((-8))+r9*8+rsp],r13
				274	mov QWORD[r9*8+rsp],rdx
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	275
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	276	lea r14,[1+r14]
				277	jmp NEAR $L$outer
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	278	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	279	$L$outer:
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	280	lea rdx,[((24+128))+r9*8+rsp]
				281	and rdx,-16
				282	pxor xmm4,xmm4
				283	pxor xmm5,xmm5
				284	movdqa xmm0,XMMWORD[((-128))+r12]
				285	movdqa xmm1,XMMWORD[((-112))+r12]
				286	movdqa xmm2,XMMWORD[((-96))+r12]
				287	movdqa xmm3,XMMWORD[((-80))+r12]
				288	pand xmm0,XMMWORD[((-128))+rdx]
				289	pand xmm1,XMMWORD[((-112))+rdx]
				290	por xmm4,xmm0
				291	pand xmm2,XMMWORD[((-96))+rdx]
				292	por xmm5,xmm1
				293	pand xmm3,XMMWORD[((-80))+rdx]
				294	por xmm4,xmm2
				295	por xmm5,xmm3
				296	movdqa xmm0,XMMWORD[((-64))+r12]
				297	movdqa xmm1,XMMWORD[((-48))+r12]
				298	movdqa xmm2,XMMWORD[((-32))+r12]
				299	movdqa xmm3,XMMWORD[((-16))+r12]
				300	pand xmm0,XMMWORD[((-64))+rdx]
				301	pand xmm1,XMMWORD[((-48))+rdx]
				302	por xmm4,xmm0
				303	pand xmm2,XMMWORD[((-32))+rdx]
				304	por xmm5,xmm1
				305	pand xmm3,XMMWORD[((-16))+rdx]
				306	por xmm4,xmm2
				307	por xmm5,xmm3
				308	movdqa xmm0,XMMWORD[r12]
				309	movdqa xmm1,XMMWORD[16+r12]
				310	movdqa xmm2,XMMWORD[32+r12]
				311	movdqa xmm3,XMMWORD[48+r12]
				312	pand xmm0,XMMWORD[rdx]
				313	pand xmm1,XMMWORD[16+rdx]
				314	por xmm4,xmm0
				315	pand xmm2,XMMWORD[32+rdx]
				316	por xmm5,xmm1
				317	pand xmm3,XMMWORD[48+rdx]
				318	por xmm4,xmm2
				319	por xmm5,xmm3
				320	movdqa xmm0,XMMWORD[64+r12]
				321	movdqa xmm1,XMMWORD[80+r12]
				322	movdqa xmm2,XMMWORD[96+r12]
				323	movdqa xmm3,XMMWORD[112+r12]
				324	pand xmm0,XMMWORD[64+rdx]
				325	pand xmm1,XMMWORD[80+rdx]
				326	por xmm4,xmm0
				327	pand xmm2,XMMWORD[96+rdx]
				328	por xmm5,xmm1
				329	pand xmm3,XMMWORD[112+rdx]
				330	por xmm4,xmm2
				331	por xmm5,xmm3
				332	por xmm4,xmm5
				333	pshufd xmm0,xmm4,0x4e
				334	por xmm0,xmm4
				335	lea r12,[256+r12]
				336
				337	mov rax,QWORD[rsi]
				338	DB 102,72,15,126,195
				339
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	340	xor r15,r15
				341	mov rbp,r8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	342	mov r10,QWORD[rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	343
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	344	mul rbx
				345	add r10,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	346	mov rax,QWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	347	adc rdx,0
				348
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	349	imul rbp,r10
				350	mov r11,rdx
				351
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	352	mul rbp
				353	add r10,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	354	mov rax,QWORD[8+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	355	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	356	mov r10,QWORD[8+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	357	mov r13,rdx
				358
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	359	lea r15,[1+r15]
				360	jmp NEAR $L$inner_enter
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	361
				362	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	363	$L$inner:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	364	add r13,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	365	mov rax,QWORD[r15*8+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	366	adc rdx,0
				367	add r13,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	368	mov r10,QWORD[r15*8+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	369	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	370	mov QWORD[((-16))+r15*8+rsp],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	371	mov r13,rdx
				372
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	373	$L$inner_enter:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	374	mul rbx
				375	add r11,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	376	mov rax,QWORD[r15*8+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	377	adc rdx,0
				378	add r10,r11
				379	mov r11,rdx
				380	adc r11,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	381	lea r15,[1+r15]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	382
				383	mul rbp
				384	cmp r15,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	385	jne NEAR $L$inner
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	386
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	387	add r13,rax
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	388	adc rdx,0
				389	add r13,r10
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	390	mov r10,QWORD[r9*8+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	391	adc rdx,0
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	392	mov QWORD[((-16))+r9*8+rsp],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	393	mov r13,rdx
				394
				395	xor rdx,rdx
				396	add r13,r11
				397	adc rdx,0
				398	add r13,r10
				399	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	400	mov QWORD[((-8))+r9*8+rsp],r13
				401	mov QWORD[r9*8+rsp],rdx
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	402
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	403	lea r14,[1+r14]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	404	cmp r14,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	405	jb NEAR $L$outer
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	406
				407	xor r14,r14
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	408	mov rax,QWORD[rsp]
				409	lea rsi,[rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	410	mov r15,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	411	jmp NEAR $L$sub
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	412	ALIGN 16
Robert Sloan	8ff0355	2017-06-14 12:40:58 -0700	[diff] [blame^]	413	$L$sub:
				414	sbb rax,QWORD[r14*8+rcx]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	415	mov QWORD[r14*8+rdi],rax
				416	mov rax,QWORD[8+r14*8+rsi]
				417	lea r14,[1+r14]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	418	dec r15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	419	jnz NEAR $L$sub
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	420
				421	sbb rax,0
				422	xor r14,r14
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	423	and rsi,rax
				424	not rax
				425	mov rcx,rdi
				426	and rcx,rax
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	427	mov r15,r9
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	428	or rsi,rcx
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	429	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	430	$L$copy:
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	431	mov rax,QWORD[r14*8+rsi]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	432	mov QWORD[r14*8+rsp],r14
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	433	mov QWORD[r14*8+rdi],rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	434	lea r14,[1+r14]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	435	sub r15,1
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	436	jnz NEAR $L$copy
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	437
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	438	mov rsi,QWORD[8+r9*8+rsp]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	439
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	440	mov rax,1
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	441
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	442	mov r15,QWORD[((-48))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	443
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	444	mov r14,QWORD[((-40))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	445
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	446	mov r13,QWORD[((-32))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	447
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	448	mov r12,QWORD[((-24))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	449
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	450	mov rbp,QWORD[((-16))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	451
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	452	mov rbx,QWORD[((-8))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	453
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	454	lea rsp,[rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	455
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	456	$L$mul_epilogue:
				457	mov rdi,QWORD[8+rsp] ;WIN64 epilogue
				458	mov rsi,QWORD[16+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	459	DB 0F3h,0C3h ;repret
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	460
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	461	$L$SEH_end_bn_mul_mont_gather5:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	462
				463	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	464	bn_mul4x_mont_gather5:
				465	mov QWORD[8+rsp],rdi ;WIN64 prologue
				466	mov QWORD[16+rsp],rsi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	467	mov rax,rsp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	468	$L$SEH_begin_bn_mul4x_mont_gather5:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	469	mov rdi,rcx
				470	mov rsi,rdx
				471	mov rdx,r8
				472	mov rcx,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	473	mov r8,QWORD[40+rsp]
				474	mov r9,QWORD[48+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	475
				476
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	477
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	478	DB 0x67
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	479	mov rax,rsp
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	480
				481	$L$mul4x_enter:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	482	push rbx
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	483
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	484	push rbp
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	485
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	486	push r12
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	487
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	488	push r13
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	489
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	490	push r14
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	491
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	492	push r15
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	493
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	494	$L$mul4x_prologue:
				495
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	496	DB 0x67
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	497	shl r9d,3
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	498	lea r10,[r9*2+r9]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	499	neg r9
				500
				501
				502
				503
				504
				505
				506
				507
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	508
				509
				510	lea r11,[((-320))+r9*2+rsp]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	511	mov rbp,rsp
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	512	sub r11,rdi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	513	and r11,4095
				514	cmp r10,r11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	515	jb NEAR $L$mul4xsp_alt
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	516	sub rbp,r11
				517	lea rbp,[((-320))+r9*2+rbp]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	518	jmp NEAR $L$mul4xsp_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	519
				520	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	521	$L$mul4xsp_alt:
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	522	lea r10,[((4096-320))+r9*2]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	523	lea rbp,[((-320))+r9*2+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	524	sub r11,r10
				525	mov r10,0
				526	cmovc r11,r10
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	527	sub rbp,r11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	528	$L$mul4xsp_done:
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	529	and rbp,-64
				530	mov r11,rsp
				531	sub r11,rbp
				532	and r11,-4096
				533	lea rsp,[rbp*1+r11]
				534	mov r10,QWORD[rsp]
				535	cmp rsp,rbp
				536	ja NEAR $L$mul4x_page_walk
				537	jmp NEAR $L$mul4x_page_walk_done
				538
				539	$L$mul4x_page_walk:
				540	lea rsp,[((-4096))+rsp]
				541	mov r10,QWORD[rsp]
				542	cmp rsp,rbp
				543	ja NEAR $L$mul4x_page_walk
				544	$L$mul4x_page_walk_done:
				545
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	546	neg r9
				547
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	548	mov QWORD[40+rsp],rax
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	549
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	550	$L$mul4x_body:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	551
				552	call mul4x_internal
				553
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	554	mov rsi,QWORD[40+rsp]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	555
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	556	mov rax,1
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	557
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	558	mov r15,QWORD[((-48))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	559
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	560	mov r14,QWORD[((-40))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	561
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	562	mov r13,QWORD[((-32))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	563
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	564	mov r12,QWORD[((-24))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	565
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	566	mov rbp,QWORD[((-16))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	567
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	568	mov rbx,QWORD[((-8))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	569
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	570	lea rsp,[rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	571
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	572	$L$mul4x_epilogue:
				573	mov rdi,QWORD[8+rsp] ;WIN64 epilogue
				574	mov rsi,QWORD[16+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	575	DB 0F3h,0C3h ;repret
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	576
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	577	$L$SEH_end_bn_mul4x_mont_gather5:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	578
				579
				580	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	581	mul4x_internal:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	582	shl r9,5
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	583	movd xmm5,DWORD[56+rax]
				584	lea rax,[$L$inc]
				585	lea r13,[128+r9*1+rdx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	586	shr r9,5
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	587	movdqa xmm0,XMMWORD[rax]
				588	movdqa xmm1,XMMWORD[16+rax]
				589	lea r10,[((88-112))+r9*1+rsp]
				590	lea r12,[128+rdx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	591
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	592	pshufd xmm5,xmm5,0
				593	movdqa xmm4,xmm1
				594	DB 0x67,0x67
				595	movdqa xmm2,xmm1
				596	paddd xmm1,xmm0
				597	pcmpeqd xmm0,xmm5
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	598	DB 0x67
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	599	movdqa xmm3,xmm4
				600	paddd xmm2,xmm1
				601	pcmpeqd xmm1,xmm5
				602	movdqa XMMWORD[112+r10],xmm0
				603	movdqa xmm0,xmm4
				604
				605	paddd xmm3,xmm2
				606	pcmpeqd xmm2,xmm5
				607	movdqa XMMWORD[128+r10],xmm1
				608	movdqa xmm1,xmm4
				609
				610	paddd xmm0,xmm3
				611	pcmpeqd xmm3,xmm5
				612	movdqa XMMWORD[144+r10],xmm2
				613	movdqa xmm2,xmm4
				614
				615	paddd xmm1,xmm0
				616	pcmpeqd xmm0,xmm5
				617	movdqa XMMWORD[160+r10],xmm3
				618	movdqa xmm3,xmm4
				619	paddd xmm2,xmm1
				620	pcmpeqd xmm1,xmm5
				621	movdqa XMMWORD[176+r10],xmm0
				622	movdqa xmm0,xmm4
				623
				624	paddd xmm3,xmm2
				625	pcmpeqd xmm2,xmm5
				626	movdqa XMMWORD[192+r10],xmm1
				627	movdqa xmm1,xmm4
				628
				629	paddd xmm0,xmm3
				630	pcmpeqd xmm3,xmm5
				631	movdqa XMMWORD[208+r10],xmm2
				632	movdqa xmm2,xmm4
				633
				634	paddd xmm1,xmm0
				635	pcmpeqd xmm0,xmm5
				636	movdqa XMMWORD[224+r10],xmm3
				637	movdqa xmm3,xmm4
				638	paddd xmm2,xmm1
				639	pcmpeqd xmm1,xmm5
				640	movdqa XMMWORD[240+r10],xmm0
				641	movdqa xmm0,xmm4
				642
				643	paddd xmm3,xmm2
				644	pcmpeqd xmm2,xmm5
				645	movdqa XMMWORD[256+r10],xmm1
				646	movdqa xmm1,xmm4
				647
				648	paddd xmm0,xmm3
				649	pcmpeqd xmm3,xmm5
				650	movdqa XMMWORD[272+r10],xmm2
				651	movdqa xmm2,xmm4
				652
				653	paddd xmm1,xmm0
				654	pcmpeqd xmm0,xmm5
				655	movdqa XMMWORD[288+r10],xmm3
				656	movdqa xmm3,xmm4
				657	paddd xmm2,xmm1
				658	pcmpeqd xmm1,xmm5
				659	movdqa XMMWORD[304+r10],xmm0
				660
				661	paddd xmm3,xmm2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	662	DB 0x67
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	663	pcmpeqd xmm2,xmm5
				664	movdqa XMMWORD[320+r10],xmm1
				665
				666	pcmpeqd xmm3,xmm5
				667	movdqa XMMWORD[336+r10],xmm2
				668	pand xmm0,XMMWORD[64+r12]
				669
				670	pand xmm1,XMMWORD[80+r12]
				671	pand xmm2,XMMWORD[96+r12]
				672	movdqa XMMWORD[352+r10],xmm3
				673	pand xmm3,XMMWORD[112+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	674	por xmm0,xmm2
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	675	por xmm1,xmm3
				676	movdqa xmm4,XMMWORD[((-128))+r12]
				677	movdqa xmm5,XMMWORD[((-112))+r12]
				678	movdqa xmm2,XMMWORD[((-96))+r12]
				679	pand xmm4,XMMWORD[112+r10]
				680	movdqa xmm3,XMMWORD[((-80))+r12]
				681	pand xmm5,XMMWORD[128+r10]
				682	por xmm0,xmm4
				683	pand xmm2,XMMWORD[144+r10]
				684	por xmm1,xmm5
				685	pand xmm3,XMMWORD[160+r10]
				686	por xmm0,xmm2
				687	por xmm1,xmm3
				688	movdqa xmm4,XMMWORD[((-64))+r12]
				689	movdqa xmm5,XMMWORD[((-48))+r12]
				690	movdqa xmm2,XMMWORD[((-32))+r12]
				691	pand xmm4,XMMWORD[176+r10]
				692	movdqa xmm3,XMMWORD[((-16))+r12]
				693	pand xmm5,XMMWORD[192+r10]
				694	por xmm0,xmm4
				695	pand xmm2,XMMWORD[208+r10]
				696	por xmm1,xmm5
				697	pand xmm3,XMMWORD[224+r10]
				698	por xmm0,xmm2
				699	por xmm1,xmm3
				700	movdqa xmm4,XMMWORD[r12]
				701	movdqa xmm5,XMMWORD[16+r12]
				702	movdqa xmm2,XMMWORD[32+r12]
				703	pand xmm4,XMMWORD[240+r10]
				704	movdqa xmm3,XMMWORD[48+r12]
				705	pand xmm5,XMMWORD[256+r10]
				706	por xmm0,xmm4
				707	pand xmm2,XMMWORD[272+r10]
				708	por xmm1,xmm5
				709	pand xmm3,XMMWORD[288+r10]
				710	por xmm0,xmm2
				711	por xmm1,xmm3
				712	por xmm0,xmm1
				713	pshufd xmm1,xmm0,0x4e
				714	por xmm0,xmm1
				715	lea r12,[256+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	716	DB 102,72,15,126,195
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	717
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	718	mov QWORD[((16+8))+rsp],r13
				719	mov QWORD[((56+8))+rsp],rdi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	720
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	721	mov r8,QWORD[r8]
				722	mov rax,QWORD[rsi]
				723	lea rsi,[r9*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	724	neg r9
				725
				726	mov rbp,r8
				727	mul rbx
				728	mov r10,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	729	mov rax,QWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	730
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	731	imul rbp,r10
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	732	lea r14,[((64+8))+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	733	mov r11,rdx
				734
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	735	mul rbp
				736	add r10,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	737	mov rax,QWORD[8+r9*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	738	adc rdx,0
				739	mov rdi,rdx
				740
				741	mul rbx
				742	add r11,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	743	mov rax,QWORD[8+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	744	adc rdx,0
				745	mov r10,rdx
				746
				747	mul rbp
				748	add rdi,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	749	mov rax,QWORD[16+r9*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	750	adc rdx,0
				751	add rdi,r11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	752	lea r15,[32+r9]
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	753	lea rcx,[32+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	754	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	755	mov QWORD[r14],rdi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	756	mov r13,rdx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	757	jmp NEAR $L$1st4x
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	758
				759	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	760	$L$1st4x:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	761	mul rbx
				762	add r10,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	763	mov rax,QWORD[((-16))+rcx]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	764	lea r14,[32+r14]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	765	adc rdx,0
				766	mov r11,rdx
				767
				768	mul rbp
				769	add r13,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	770	mov rax,QWORD[((-8))+r15*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	771	adc rdx,0
				772	add r13,r10
				773	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	774	mov QWORD[((-24))+r14],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	775	mov rdi,rdx
				776
				777	mul rbx
				778	add r11,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	779	mov rax,QWORD[((-8))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	780	adc rdx,0
				781	mov r10,rdx
				782
				783	mul rbp
				784	add rdi,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	785	mov rax,QWORD[r15*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	786	adc rdx,0
				787	add rdi,r11
				788	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	789	mov QWORD[((-16))+r14],rdi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	790	mov r13,rdx
				791
				792	mul rbx
				793	add r10,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	794	mov rax,QWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	795	adc rdx,0
				796	mov r11,rdx
				797
				798	mul rbp
				799	add r13,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	800	mov rax,QWORD[8+r15*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	801	adc rdx,0
				802	add r13,r10
				803	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	804	mov QWORD[((-8))+r14],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	805	mov rdi,rdx
				806
				807	mul rbx
				808	add r11,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	809	mov rax,QWORD[8+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	810	adc rdx,0
				811	mov r10,rdx
				812
				813	mul rbp
				814	add rdi,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	815	mov rax,QWORD[16+r15*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	816	adc rdx,0
				817	add rdi,r11
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	818	lea rcx,[32+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	819	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	820	mov QWORD[r14],rdi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	821	mov r13,rdx
				822
				823	add r15,32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	824	jnz NEAR $L$1st4x
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	825
				826	mul rbx
				827	add r10,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	828	mov rax,QWORD[((-16))+rcx]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	829	lea r14,[32+r14]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	830	adc rdx,0
				831	mov r11,rdx
				832
				833	mul rbp
				834	add r13,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	835	mov rax,QWORD[((-8))+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	836	adc rdx,0
				837	add r13,r10
				838	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	839	mov QWORD[((-24))+r14],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	840	mov rdi,rdx
				841
				842	mul rbx
				843	add r11,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	844	mov rax,QWORD[((-8))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	845	adc rdx,0
				846	mov r10,rdx
				847
				848	mul rbp
				849	add rdi,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	850	mov rax,QWORD[r9*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	851	adc rdx,0
				852	add rdi,r11
				853	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	854	mov QWORD[((-16))+r14],rdi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	855	mov r13,rdx
				856
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	857	lea rcx,[r9*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	858
				859	xor rdi,rdi
				860	add r13,r10
				861	adc rdi,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	862	mov QWORD[((-8))+r14],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	863
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	864	jmp NEAR $L$outer4x
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	865
				866	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	867	$L$outer4x:
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	868	lea rdx,[((16+128))+r14]
				869	pxor xmm4,xmm4
				870	pxor xmm5,xmm5
				871	movdqa xmm0,XMMWORD[((-128))+r12]
				872	movdqa xmm1,XMMWORD[((-112))+r12]
				873	movdqa xmm2,XMMWORD[((-96))+r12]
				874	movdqa xmm3,XMMWORD[((-80))+r12]
				875	pand xmm0,XMMWORD[((-128))+rdx]
				876	pand xmm1,XMMWORD[((-112))+rdx]
				877	por xmm4,xmm0
				878	pand xmm2,XMMWORD[((-96))+rdx]
				879	por xmm5,xmm1
				880	pand xmm3,XMMWORD[((-80))+rdx]
				881	por xmm4,xmm2
				882	por xmm5,xmm3
				883	movdqa xmm0,XMMWORD[((-64))+r12]
				884	movdqa xmm1,XMMWORD[((-48))+r12]
				885	movdqa xmm2,XMMWORD[((-32))+r12]
				886	movdqa xmm3,XMMWORD[((-16))+r12]
				887	pand xmm0,XMMWORD[((-64))+rdx]
				888	pand xmm1,XMMWORD[((-48))+rdx]
				889	por xmm4,xmm0
				890	pand xmm2,XMMWORD[((-32))+rdx]
				891	por xmm5,xmm1
				892	pand xmm3,XMMWORD[((-16))+rdx]
				893	por xmm4,xmm2
				894	por xmm5,xmm3
				895	movdqa xmm0,XMMWORD[r12]
				896	movdqa xmm1,XMMWORD[16+r12]
				897	movdqa xmm2,XMMWORD[32+r12]
				898	movdqa xmm3,XMMWORD[48+r12]
				899	pand xmm0,XMMWORD[rdx]
				900	pand xmm1,XMMWORD[16+rdx]
				901	por xmm4,xmm0
				902	pand xmm2,XMMWORD[32+rdx]
				903	por xmm5,xmm1
				904	pand xmm3,XMMWORD[48+rdx]
				905	por xmm4,xmm2
				906	por xmm5,xmm3
				907	movdqa xmm0,XMMWORD[64+r12]
				908	movdqa xmm1,XMMWORD[80+r12]
				909	movdqa xmm2,XMMWORD[96+r12]
				910	movdqa xmm3,XMMWORD[112+r12]
				911	pand xmm0,XMMWORD[64+rdx]
				912	pand xmm1,XMMWORD[80+rdx]
				913	por xmm4,xmm0
				914	pand xmm2,XMMWORD[96+rdx]
				915	por xmm5,xmm1
				916	pand xmm3,XMMWORD[112+rdx]
				917	por xmm4,xmm2
				918	por xmm5,xmm3
				919	por xmm4,xmm5
				920	pshufd xmm0,xmm4,0x4e
				921	por xmm0,xmm4
				922	lea r12,[256+r12]
				923	DB 102,72,15,126,195
				924
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	925	mov r10,QWORD[r9*1+r14]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	926	mov rbp,r8
				927	mul rbx
				928	add r10,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	929	mov rax,QWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	930	adc rdx,0
				931
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	932	imul rbp,r10
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	933	mov r11,rdx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	934	mov QWORD[r14],rdi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	935
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	936	lea r14,[r9*1+r14]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	937
				938	mul rbp
				939	add r10,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	940	mov rax,QWORD[8+r9*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	941	adc rdx,0
				942	mov rdi,rdx
				943
				944	mul rbx
				945	add r11,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	946	mov rax,QWORD[8+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	947	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	948	add r11,QWORD[8+r14]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	949	adc rdx,0
				950	mov r10,rdx
				951
				952	mul rbp
				953	add rdi,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	954	mov rax,QWORD[16+r9*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	955	adc rdx,0
				956	add rdi,r11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	957	lea r15,[32+r9]
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	958	lea rcx,[32+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	959	adc rdx,0
				960	mov r13,rdx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	961	jmp NEAR $L$inner4x
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	962
				963	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	964	$L$inner4x:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	965	mul rbx
				966	add r10,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	967	mov rax,QWORD[((-16))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	968	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	969	add r10,QWORD[16+r14]
				970	lea r14,[32+r14]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	971	adc rdx,0
				972	mov r11,rdx
				973
				974	mul rbp
				975	add r13,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	976	mov rax,QWORD[((-8))+r15*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	977	adc rdx,0
				978	add r13,r10
				979	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	980	mov QWORD[((-32))+r14],rdi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	981	mov rdi,rdx
				982
				983	mul rbx
				984	add r11,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	985	mov rax,QWORD[((-8))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	986	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	987	add r11,QWORD[((-8))+r14]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	988	adc rdx,0
				989	mov r10,rdx
				990
				991	mul rbp
				992	add rdi,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	993	mov rax,QWORD[r15*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	994	adc rdx,0
				995	add rdi,r11
				996	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	997	mov QWORD[((-24))+r14],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	998	mov r13,rdx
				999
				1000	mul rbx
				1001	add r10,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1002	mov rax,QWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1003	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1004	add r10,QWORD[r14]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1005	adc rdx,0
				1006	mov r11,rdx
				1007
				1008	mul rbp
				1009	add r13,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1010	mov rax,QWORD[8+r15*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1011	adc rdx,0
				1012	add r13,r10
				1013	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1014	mov QWORD[((-16))+r14],rdi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1015	mov rdi,rdx
				1016
				1017	mul rbx
				1018	add r11,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1019	mov rax,QWORD[8+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1020	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1021	add r11,QWORD[8+r14]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1022	adc rdx,0
				1023	mov r10,rdx
				1024
				1025	mul rbp
				1026	add rdi,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1027	mov rax,QWORD[16+r15*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1028	adc rdx,0
				1029	add rdi,r11
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1030	lea rcx,[32+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1031	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1032	mov QWORD[((-8))+r14],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1033	mov r13,rdx
				1034
				1035	add r15,32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1036	jnz NEAR $L$inner4x
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1037
				1038	mul rbx
				1039	add r10,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1040	mov rax,QWORD[((-16))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1041	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1042	add r10,QWORD[16+r14]
				1043	lea r14,[32+r14]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1044	adc rdx,0
				1045	mov r11,rdx
				1046
				1047	mul rbp
				1048	add r13,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1049	mov rax,QWORD[((-8))+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1050	adc rdx,0
				1051	add r13,r10
				1052	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1053	mov QWORD[((-32))+r14],rdi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1054	mov rdi,rdx
				1055
				1056	mul rbx
				1057	add r11,rax
				1058	mov rax,rbp
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1059	mov rbp,QWORD[((-8))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1060	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1061	add r11,QWORD[((-8))+r14]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1062	adc rdx,0
				1063	mov r10,rdx
				1064
				1065	mul rbp
				1066	add rdi,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1067	mov rax,QWORD[r9*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1068	adc rdx,0
				1069	add rdi,r11
				1070	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1071	mov QWORD[((-24))+r14],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1072	mov r13,rdx
				1073
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1074	mov QWORD[((-16))+r14],rdi
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1075	lea rcx,[r9*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1076
				1077	xor rdi,rdi
				1078	add r13,r10
				1079	adc rdi,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1080	add r13,QWORD[r14]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1081	adc rdi,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1082	mov QWORD[((-8))+r14],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1083
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1084	cmp r12,QWORD[((16+8))+rsp]
				1085	jb NEAR $L$outer4x
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1086	xor rax,rax
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1087	sub rbp,r13
				1088	adc r15,r15
				1089	or rdi,r15
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1090	sub rax,rdi
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1091	lea rbx,[r9*1+r14]
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1092	mov r12,QWORD[rcx]
				1093	lea rbp,[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1094	mov rcx,r9
				1095	sar rcx,3+2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1096	mov rdi,QWORD[((56+8))+rsp]
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1097	dec r12
				1098	xor r10,r10
				1099	mov r13,QWORD[8+rbp]
				1100	mov r14,QWORD[16+rbp]
				1101	mov r15,QWORD[24+rbp]
				1102	jmp NEAR $L$sqr4x_sub_entry
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1103
				1104	global bn_power5
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1105
				1106	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1107	bn_power5:
				1108	mov QWORD[8+rsp],rdi ;WIN64 prologue
				1109	mov QWORD[16+rsp],rsi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1110	mov rax,rsp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1111	$L$SEH_begin_bn_power5:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1112	mov rdi,rcx
				1113	mov rsi,rdx
				1114	mov rdx,r8
				1115	mov rcx,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1116	mov r8,QWORD[40+rsp]
				1117	mov r9,QWORD[48+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1118
				1119
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1120
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1121	mov rax,rsp
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1122
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1123	push rbx
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1124
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1125	push rbp
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1126
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1127	push r12
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1128
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1129	push r13
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1130
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1131	push r14
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1132
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1133	push r15
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1134
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1135	$L$power5_prologue:
				1136
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1137	shl r9d,3
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1138	lea r10d,[r9*2+r9]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1139	neg r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1140	mov r8,QWORD[r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1141
				1142
				1143
				1144
				1145
				1146
				1147
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1148
				1149	lea r11,[((-320))+r9*2+rsp]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1150	mov rbp,rsp
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1151	sub r11,rdi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1152	and r11,4095
				1153	cmp r10,r11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1154	jb NEAR $L$pwr_sp_alt
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1155	sub rbp,r11
				1156	lea rbp,[((-320))+r9*2+rbp]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1157	jmp NEAR $L$pwr_sp_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1158
				1159	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1160	$L$pwr_sp_alt:
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1161	lea r10,[((4096-320))+r9*2]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1162	lea rbp,[((-320))+r9*2+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1163	sub r11,r10
				1164	mov r10,0
				1165	cmovc r11,r10
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1166	sub rbp,r11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1167	$L$pwr_sp_done:
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1168	and rbp,-64
				1169	mov r11,rsp
				1170	sub r11,rbp
				1171	and r11,-4096
				1172	lea rsp,[rbp*1+r11]
				1173	mov r10,QWORD[rsp]
				1174	cmp rsp,rbp
				1175	ja NEAR $L$pwr_page_walk
				1176	jmp NEAR $L$pwr_page_walk_done
				1177
				1178	$L$pwr_page_walk:
				1179	lea rsp,[((-4096))+rsp]
				1180	mov r10,QWORD[rsp]
				1181	cmp rsp,rbp
				1182	ja NEAR $L$pwr_page_walk
				1183	$L$pwr_page_walk_done:
				1184
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1185	mov r10,r9
				1186	neg r9
				1187
				1188
				1189
				1190
				1191
				1192
				1193
				1194
				1195
				1196
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1197	mov QWORD[32+rsp],r8
				1198	mov QWORD[40+rsp],rax
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1199
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1200	$L$power5_body:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1201	DB 102,72,15,110,207
				1202	DB 102,72,15,110,209
				1203	DB 102,73,15,110,218
				1204	DB 102,72,15,110,226
				1205
				1206	call __bn_sqr8x_internal
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1207	call __bn_post4x_internal
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1208	call __bn_sqr8x_internal
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1209	call __bn_post4x_internal
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1210	call __bn_sqr8x_internal
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1211	call __bn_post4x_internal
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1212	call __bn_sqr8x_internal
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1213	call __bn_post4x_internal
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1214	call __bn_sqr8x_internal
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1215	call __bn_post4x_internal
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1216
				1217	DB 102,72,15,126,209
				1218	DB 102,72,15,126,226
				1219	mov rdi,rsi
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1220	mov rax,QWORD[40+rsp]
				1221	lea r8,[32+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1222
				1223	call mul4x_internal
				1224
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1225	mov rsi,QWORD[40+rsp]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1226
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1227	mov rax,1
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1228	mov r15,QWORD[((-48))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1229
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1230	mov r14,QWORD[((-40))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1231
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1232	mov r13,QWORD[((-32))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1233
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1234	mov r12,QWORD[((-24))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1235
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1236	mov rbp,QWORD[((-16))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1237
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1238	mov rbx,QWORD[((-8))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1239
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1240	lea rsp,[rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1241
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1242	$L$power5_epilogue:
				1243	mov rdi,QWORD[8+rsp] ;WIN64 epilogue
				1244	mov rsi,QWORD[16+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1245	DB 0F3h,0C3h ;repret
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1246
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1247	$L$SEH_end_bn_power5:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1248
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1249	global bn_sqr8x_internal
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1250
				1251
				1252	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1253	bn_sqr8x_internal:
				1254	__bn_sqr8x_internal:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1255
				1256
				1257
				1258
				1259
				1260
				1261
				1262
				1263
				1264
				1265
				1266
				1267
				1268
				1269
				1270
				1271
				1272
				1273
				1274
				1275
				1276
				1277
				1278
				1279
				1280
				1281
				1282
				1283
				1284
				1285
				1286
				1287
				1288
				1289
				1290
				1291
				1292
				1293
				1294
				1295
				1296
				1297
				1298
				1299
				1300
				1301
				1302
				1303
				1304
				1305
				1306
				1307
				1308
				1309
				1310
				1311
				1312
				1313
				1314
				1315
				1316
				1317
				1318
				1319
				1320
				1321
				1322
				1323
				1324
				1325
				1326
				1327
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1328	lea rbp,[32+r10]
				1329	lea rsi,[r9*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1330
				1331	mov rcx,r9
				1332
				1333
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1334	mov r14,QWORD[((-32))+rbp*1+rsi]
				1335	lea rdi,[((48+8))+r9*2+rsp]
				1336	mov rax,QWORD[((-24))+rbp*1+rsi]
				1337	lea rdi,[((-32))+rbp*1+rdi]
				1338	mov rbx,QWORD[((-16))+rbp*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1339	mov r15,rax
				1340
				1341	mul r14
				1342	mov r10,rax
				1343	mov rax,rbx
				1344	mov r11,rdx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1345	mov QWORD[((-24))+rbp*1+rdi],r10
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1346
				1347	mul r14
				1348	add r11,rax
				1349	mov rax,rbx
				1350	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1351	mov QWORD[((-16))+rbp*1+rdi],r11
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1352	mov r10,rdx
				1353
				1354
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1355	mov rbx,QWORD[((-8))+rbp*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1356	mul r15
				1357	mov r12,rax
				1358	mov rax,rbx
				1359	mov r13,rdx
				1360
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1361	lea rcx,[rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1362	mul r14
				1363	add r10,rax
				1364	mov rax,rbx
				1365	mov r11,rdx
				1366	adc r11,0
				1367	add r10,r12
				1368	adc r11,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1369	mov QWORD[((-8))+rcx*1+rdi],r10
				1370	jmp NEAR $L$sqr4x_1st
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1371
				1372	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1373	$L$sqr4x_1st:
				1374	mov rbx,QWORD[rcx*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1375	mul r15
				1376	add r13,rax
				1377	mov rax,rbx
				1378	mov r12,rdx
				1379	adc r12,0
				1380
				1381	mul r14
				1382	add r11,rax
				1383	mov rax,rbx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1384	mov rbx,QWORD[8+rcx*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1385	mov r10,rdx
				1386	adc r10,0
				1387	add r11,r13
				1388	adc r10,0
				1389
				1390
				1391	mul r15
				1392	add r12,rax
				1393	mov rax,rbx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1394	mov QWORD[rcx*1+rdi],r11
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1395	mov r13,rdx
				1396	adc r13,0
				1397
				1398	mul r14
				1399	add r10,rax
				1400	mov rax,rbx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1401	mov rbx,QWORD[16+rcx*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1402	mov r11,rdx
				1403	adc r11,0
				1404	add r10,r12
				1405	adc r11,0
				1406
				1407	mul r15
				1408	add r13,rax
				1409	mov rax,rbx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1410	mov QWORD[8+rcx*1+rdi],r10
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1411	mov r12,rdx
				1412	adc r12,0
				1413
				1414	mul r14
				1415	add r11,rax
				1416	mov rax,rbx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1417	mov rbx,QWORD[24+rcx*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1418	mov r10,rdx
				1419	adc r10,0
				1420	add r11,r13
				1421	adc r10,0
				1422
				1423
				1424	mul r15
				1425	add r12,rax
				1426	mov rax,rbx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1427	mov QWORD[16+rcx*1+rdi],r11
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1428	mov r13,rdx
				1429	adc r13,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1430	lea rcx,[32+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1431
				1432	mul r14
				1433	add r10,rax
				1434	mov rax,rbx
				1435	mov r11,rdx
				1436	adc r11,0
				1437	add r10,r12
				1438	adc r11,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1439	mov QWORD[((-8))+rcx*1+rdi],r10
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1440
				1441	cmp rcx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1442	jne NEAR $L$sqr4x_1st
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1443
				1444	mul r15
				1445	add r13,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1446	lea rbp,[16+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1447	adc rdx,0
				1448	add r13,r11
				1449	adc rdx,0
				1450
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1451	mov QWORD[rdi],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1452	mov r12,rdx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1453	mov QWORD[8+rdi],rdx
				1454	jmp NEAR $L$sqr4x_outer
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1455
				1456	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1457	$L$sqr4x_outer:
				1458	mov r14,QWORD[((-32))+rbp*1+rsi]
				1459	lea rdi,[((48+8))+r9*2+rsp]
				1460	mov rax,QWORD[((-24))+rbp*1+rsi]
				1461	lea rdi,[((-32))+rbp*1+rdi]
				1462	mov rbx,QWORD[((-16))+rbp*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1463	mov r15,rax
				1464
				1465	mul r14
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1466	mov r10,QWORD[((-24))+rbp*1+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1467	add r10,rax
				1468	mov rax,rbx
				1469	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1470	mov QWORD[((-24))+rbp*1+rdi],r10
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1471	mov r11,rdx
				1472
				1473	mul r14
				1474	add r11,rax
				1475	mov rax,rbx
				1476	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1477	add r11,QWORD[((-16))+rbp*1+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1478	mov r10,rdx
				1479	adc r10,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1480	mov QWORD[((-16))+rbp*1+rdi],r11
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1481
				1482	xor r12,r12
				1483
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1484	mov rbx,QWORD[((-8))+rbp*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1485	mul r15
				1486	add r12,rax
				1487	mov rax,rbx
				1488	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1489	add r12,QWORD[((-8))+rbp*1+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1490	mov r13,rdx
				1491	adc r13,0
				1492
				1493	mul r14
				1494	add r10,rax
				1495	mov rax,rbx
				1496	adc rdx,0
				1497	add r10,r12
				1498	mov r11,rdx
				1499	adc r11,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1500	mov QWORD[((-8))+rbp*1+rdi],r10
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1501
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1502	lea rcx,[rbp]
				1503	jmp NEAR $L$sqr4x_inner
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1504
				1505	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1506	$L$sqr4x_inner:
				1507	mov rbx,QWORD[rcx*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1508	mul r15
				1509	add r13,rax
				1510	mov rax,rbx
				1511	mov r12,rdx
				1512	adc r12,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1513	add r13,QWORD[rcx*1+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1514	adc r12,0
				1515
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1516	DB 0x67
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1517	mul r14
				1518	add r11,rax
				1519	mov rax,rbx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1520	mov rbx,QWORD[8+rcx*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1521	mov r10,rdx
				1522	adc r10,0
				1523	add r11,r13
				1524	adc r10,0
				1525
				1526	mul r15
				1527	add r12,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1528	mov QWORD[rcx*1+rdi],r11
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1529	mov rax,rbx
				1530	mov r13,rdx
				1531	adc r13,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1532	add r12,QWORD[8+rcx*1+rdi]
				1533	lea rcx,[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1534	adc r13,0
				1535
				1536	mul r14
				1537	add r10,rax
				1538	mov rax,rbx
				1539	adc rdx,0
				1540	add r10,r12
				1541	mov r11,rdx
				1542	adc r11,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1543	mov QWORD[((-8))+rcx*1+rdi],r10
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1544
				1545	cmp rcx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1546	jne NEAR $L$sqr4x_inner
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1547
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1548	DB 0x67
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1549	mul r15
				1550	add r13,rax
				1551	adc rdx,0
				1552	add r13,r11
				1553	adc rdx,0
				1554
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1555	mov QWORD[rdi],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1556	mov r12,rdx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1557	mov QWORD[8+rdi],rdx
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1558
				1559	add rbp,16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1560	jnz NEAR $L$sqr4x_outer
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1561
				1562
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1563	mov r14,QWORD[((-32))+rsi]
				1564	lea rdi,[((48+8))+r9*2+rsp]
				1565	mov rax,QWORD[((-24))+rsi]
				1566	lea rdi,[((-32))+rbp*1+rdi]
				1567	mov rbx,QWORD[((-16))+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1568	mov r15,rax
				1569
				1570	mul r14
				1571	add r10,rax
				1572	mov rax,rbx
				1573	mov r11,rdx
				1574	adc r11,0
				1575
				1576	mul r14
				1577	add r11,rax
				1578	mov rax,rbx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1579	mov QWORD[((-24))+rdi],r10
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1580	mov r10,rdx
				1581	adc r10,0
				1582	add r11,r13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1583	mov rbx,QWORD[((-8))+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1584	adc r10,0
				1585
				1586	mul r15
				1587	add r12,rax
				1588	mov rax,rbx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1589	mov QWORD[((-16))+rdi],r11
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1590	mov r13,rdx
				1591	adc r13,0
				1592
				1593	mul r14
				1594	add r10,rax
				1595	mov rax,rbx
				1596	mov r11,rdx
				1597	adc r11,0
				1598	add r10,r12
				1599	adc r11,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1600	mov QWORD[((-8))+rdi],r10
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1601
				1602	mul r15
				1603	add r13,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1604	mov rax,QWORD[((-16))+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1605	adc rdx,0
				1606	add r13,r11
				1607	adc rdx,0
				1608
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1609	mov QWORD[rdi],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1610	mov r12,rdx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1611	mov QWORD[8+rdi],rdx
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1612
				1613	mul rbx
				1614	add rbp,16
				1615	xor r14,r14
				1616	sub rbp,r9
				1617	xor r15,r15
				1618
				1619	add rax,r12
				1620	adc rdx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1621	mov QWORD[8+rdi],rax
				1622	mov QWORD[16+rdi],rdx
				1623	mov QWORD[24+rdi],r15
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1624
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1625	mov rax,QWORD[((-16))+rbp*1+rsi]
				1626	lea rdi,[((48+8))+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1627	xor r10,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1628	mov r11,QWORD[8+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1629
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1630	lea r12,[r10*2+r14]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1631	shr r10,63
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1632	lea r13,[r11*2+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1633	shr r11,63
				1634	or r13,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1635	mov r10,QWORD[16+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1636	mov r14,r11
				1637	mul rax
				1638	neg r15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1639	mov r11,QWORD[24+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1640	adc r12,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1641	mov rax,QWORD[((-8))+rbp*1+rsi]
				1642	mov QWORD[rdi],r12
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1643	adc r13,rdx
				1644
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1645	lea rbx,[r10*2+r14]
				1646	mov QWORD[8+rdi],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1647	sbb r15,r15
				1648	shr r10,63
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1649	lea r8,[r11*2+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1650	shr r11,63
				1651	or r8,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1652	mov r10,QWORD[32+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1653	mov r14,r11
				1654	mul rax
				1655	neg r15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1656	mov r11,QWORD[40+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1657	adc rbx,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1658	mov rax,QWORD[rbp*1+rsi]
				1659	mov QWORD[16+rdi],rbx
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1660	adc r8,rdx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1661	lea rbp,[16+rbp]
				1662	mov QWORD[24+rdi],r8
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1663	sbb r15,r15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1664	lea rdi,[64+rdi]
				1665	jmp NEAR $L$sqr4x_shift_n_add
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1666
				1667	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1668	$L$sqr4x_shift_n_add:
				1669	lea r12,[r10*2+r14]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1670	shr r10,63
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1671	lea r13,[r11*2+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1672	shr r11,63
				1673	or r13,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1674	mov r10,QWORD[((-16))+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1675	mov r14,r11
				1676	mul rax
				1677	neg r15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1678	mov r11,QWORD[((-8))+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1679	adc r12,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1680	mov rax,QWORD[((-8))+rbp*1+rsi]
				1681	mov QWORD[((-32))+rdi],r12
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1682	adc r13,rdx
				1683
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1684	lea rbx,[r10*2+r14]
				1685	mov QWORD[((-24))+rdi],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1686	sbb r15,r15
				1687	shr r10,63
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1688	lea r8,[r11*2+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1689	shr r11,63
				1690	or r8,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1691	mov r10,QWORD[rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1692	mov r14,r11
				1693	mul rax
				1694	neg r15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1695	mov r11,QWORD[8+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1696	adc rbx,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1697	mov rax,QWORD[rbp*1+rsi]
				1698	mov QWORD[((-16))+rdi],rbx
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1699	adc r8,rdx
				1700
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1701	lea r12,[r10*2+r14]
				1702	mov QWORD[((-8))+rdi],r8
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1703	sbb r15,r15
				1704	shr r10,63
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1705	lea r13,[r11*2+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1706	shr r11,63
				1707	or r13,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1708	mov r10,QWORD[16+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1709	mov r14,r11
				1710	mul rax
				1711	neg r15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1712	mov r11,QWORD[24+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1713	adc r12,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1714	mov rax,QWORD[8+rbp*1+rsi]
				1715	mov QWORD[rdi],r12
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1716	adc r13,rdx
				1717
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1718	lea rbx,[r10*2+r14]
				1719	mov QWORD[8+rdi],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1720	sbb r15,r15
				1721	shr r10,63
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1722	lea r8,[r11*2+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1723	shr r11,63
				1724	or r8,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1725	mov r10,QWORD[32+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1726	mov r14,r11
				1727	mul rax
				1728	neg r15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1729	mov r11,QWORD[40+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1730	adc rbx,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1731	mov rax,QWORD[16+rbp*1+rsi]
				1732	mov QWORD[16+rdi],rbx
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1733	adc r8,rdx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1734	mov QWORD[24+rdi],r8
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1735	sbb r15,r15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1736	lea rdi,[64+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1737	add rbp,32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1738	jnz NEAR $L$sqr4x_shift_n_add
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1739
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1740	lea r12,[r10*2+r14]
				1741	DB 0x67
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1742	shr r10,63
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1743	lea r13,[r11*2+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1744	shr r11,63
				1745	or r13,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1746	mov r10,QWORD[((-16))+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1747	mov r14,r11
				1748	mul rax
				1749	neg r15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1750	mov r11,QWORD[((-8))+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1751	adc r12,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1752	mov rax,QWORD[((-8))+rsi]
				1753	mov QWORD[((-32))+rdi],r12
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1754	adc r13,rdx
				1755
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1756	lea rbx,[r10*2+r14]
				1757	mov QWORD[((-24))+rdi],r13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1758	sbb r15,r15
				1759	shr r10,63
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1760	lea r8,[r11*2+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1761	shr r11,63
				1762	or r8,r10
				1763	mul rax
				1764	neg r15
				1765	adc rbx,rax
				1766	adc r8,rdx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1767	mov QWORD[((-16))+rdi],rbx
				1768	mov QWORD[((-8))+rdi],r8
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1769	DB 102,72,15,126,213
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1770	__bn_sqr8x_reduction:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1771	xor rax,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1772	lea rcx,[rbp*1+r9]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1773	lea rdx,[((48+8))+r9*2+rsp]
				1774	mov QWORD[((0+8))+rsp],rcx
				1775	lea rdi,[((48+8))+r9*1+rsp]
				1776	mov QWORD[((8+8))+rsp],rdx
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1777	neg r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1778	jmp NEAR $L$8x_reduction_loop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1779
				1780	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1781	$L$8x_reduction_loop:
				1782	lea rdi,[r9*1+rdi]
				1783	DB 0x66
				1784	mov rbx,QWORD[rdi]
				1785	mov r9,QWORD[8+rdi]
				1786	mov r10,QWORD[16+rdi]
				1787	mov r11,QWORD[24+rdi]
				1788	mov r12,QWORD[32+rdi]
				1789	mov r13,QWORD[40+rdi]
				1790	mov r14,QWORD[48+rdi]
				1791	mov r15,QWORD[56+rdi]
				1792	mov QWORD[rdx],rax
				1793	lea rdi,[64+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1794
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1795	DB 0x67
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1796	mov r8,rbx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1797	imul rbx,QWORD[((32+8))+rsp]
				1798	mov rax,QWORD[rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1799	mov ecx,8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1800	jmp NEAR $L$8x_reduce
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1801
				1802	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1803	$L$8x_reduce:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1804	mul rbx
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1805	mov rax,QWORD[8+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1806	neg r8
				1807	mov r8,rdx
				1808	adc r8,0
				1809
				1810	mul rbx
				1811	add r9,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1812	mov rax,QWORD[16+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1813	adc rdx,0
				1814	add r8,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1815	mov QWORD[((48-8+8))+rcx*8+rsp],rbx
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1816	mov r9,rdx
				1817	adc r9,0
				1818
				1819	mul rbx
				1820	add r10,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1821	mov rax,QWORD[24+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1822	adc rdx,0
				1823	add r9,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1824	mov rsi,QWORD[((32+8))+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1825	mov r10,rdx
				1826	adc r10,0
				1827
				1828	mul rbx
				1829	add r11,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1830	mov rax,QWORD[32+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1831	adc rdx,0
				1832	imul rsi,r8
				1833	add r10,r11
				1834	mov r11,rdx
				1835	adc r11,0
				1836
				1837	mul rbx
				1838	add r12,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1839	mov rax,QWORD[40+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1840	adc rdx,0
				1841	add r11,r12
				1842	mov r12,rdx
				1843	adc r12,0
				1844
				1845	mul rbx
				1846	add r13,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1847	mov rax,QWORD[48+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1848	adc rdx,0
				1849	add r12,r13
				1850	mov r13,rdx
				1851	adc r13,0
				1852
				1853	mul rbx
				1854	add r14,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1855	mov rax,QWORD[56+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1856	adc rdx,0
				1857	add r13,r14
				1858	mov r14,rdx
				1859	adc r14,0
				1860
				1861	mul rbx
				1862	mov rbx,rsi
				1863	add r15,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1864	mov rax,QWORD[rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1865	adc rdx,0
				1866	add r14,r15
				1867	mov r15,rdx
				1868	adc r15,0
				1869
				1870	dec ecx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1871	jnz NEAR $L$8x_reduce
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1872
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1873	lea rbp,[64+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1874	xor rax,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1875	mov rdx,QWORD[((8+8))+rsp]
				1876	cmp rbp,QWORD[((0+8))+rsp]
				1877	jae NEAR $L$8x_no_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1878
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1879	DB 0x66
				1880	add r8,QWORD[rdi]
				1881	adc r9,QWORD[8+rdi]
				1882	adc r10,QWORD[16+rdi]
				1883	adc r11,QWORD[24+rdi]
				1884	adc r12,QWORD[32+rdi]
				1885	adc r13,QWORD[40+rdi]
				1886	adc r14,QWORD[48+rdi]
				1887	adc r15,QWORD[56+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1888	sbb rsi,rsi
				1889
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1890	mov rbx,QWORD[((48+56+8))+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1891	mov ecx,8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1892	mov rax,QWORD[rbp]
				1893	jmp NEAR $L$8x_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1894
				1895	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1896	$L$8x_tail:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1897	mul rbx
				1898	add r8,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1899	mov rax,QWORD[8+rbp]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1900	mov QWORD[rdi],r8
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1901	mov r8,rdx
				1902	adc r8,0
				1903
				1904	mul rbx
				1905	add r9,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1906	mov rax,QWORD[16+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1907	adc rdx,0
				1908	add r8,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1909	lea rdi,[8+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1910	mov r9,rdx
				1911	adc r9,0
				1912
				1913	mul rbx
				1914	add r10,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1915	mov rax,QWORD[24+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1916	adc rdx,0
				1917	add r9,r10
				1918	mov r10,rdx
				1919	adc r10,0
				1920
				1921	mul rbx
				1922	add r11,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1923	mov rax,QWORD[32+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1924	adc rdx,0
				1925	add r10,r11
				1926	mov r11,rdx
				1927	adc r11,0
				1928
				1929	mul rbx
				1930	add r12,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1931	mov rax,QWORD[40+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1932	adc rdx,0
				1933	add r11,r12
				1934	mov r12,rdx
				1935	adc r12,0
				1936
				1937	mul rbx
				1938	add r13,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1939	mov rax,QWORD[48+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1940	adc rdx,0
				1941	add r12,r13
				1942	mov r13,rdx
				1943	adc r13,0
				1944
				1945	mul rbx
				1946	add r14,rax
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1947	mov rax,QWORD[56+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1948	adc rdx,0
				1949	add r13,r14
				1950	mov r14,rdx
				1951	adc r14,0
				1952
				1953	mul rbx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1954	mov rbx,QWORD[((48-16+8))+rcx*8+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1955	add r15,rax
				1956	adc rdx,0
				1957	add r14,r15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1958	mov rax,QWORD[rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1959	mov r15,rdx
				1960	adc r15,0
				1961
				1962	dec ecx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1963	jnz NEAR $L$8x_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1964
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	1965	lea rbp,[64+rbp]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1966	mov rdx,QWORD[((8+8))+rsp]
				1967	cmp rbp,QWORD[((0+8))+rsp]
				1968	jae NEAR $L$8x_tail_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1969
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1970	mov rbx,QWORD[((48+56+8))+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1971	neg rsi
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1972	mov rax,QWORD[rbp]
				1973	adc r8,QWORD[rdi]
				1974	adc r9,QWORD[8+rdi]
				1975	adc r10,QWORD[16+rdi]
				1976	adc r11,QWORD[24+rdi]
				1977	adc r12,QWORD[32+rdi]
				1978	adc r13,QWORD[40+rdi]
				1979	adc r14,QWORD[48+rdi]
				1980	adc r15,QWORD[56+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1981	sbb rsi,rsi
				1982
				1983	mov ecx,8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1984	jmp NEAR $L$8x_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1985
				1986	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1987	$L$8x_tail_done:
Robert Sloan	4d1ac50	2017-02-06 08:36:14 -0800	[diff] [blame]	1988	xor rax,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1989	add r8,QWORD[rdx]
Adam Langley	4139edb	2016-01-13 15:00:54 -0800	[diff] [blame]	1990	adc r9,0
				1991	adc r10,0
				1992	adc r11,0
				1993	adc r12,0
				1994	adc r13,0
				1995	adc r14,0
				1996	adc r15,0
Robert Sloan	4d1ac50	2017-02-06 08:36:14 -0800	[diff] [blame]	1997	adc rax,0
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1998
				1999	neg rsi
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2000	$L$8x_no_tail:
				2001	adc r8,QWORD[rdi]
				2002	adc r9,QWORD[8+rdi]
				2003	adc r10,QWORD[16+rdi]
				2004	adc r11,QWORD[24+rdi]
				2005	adc r12,QWORD[32+rdi]
				2006	adc r13,QWORD[40+rdi]
				2007	adc r14,QWORD[48+rdi]
				2008	adc r15,QWORD[56+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2009	adc rax,0
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2010	mov rcx,QWORD[((-8))+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2011	xor rsi,rsi
				2012
				2013	DB 102,72,15,126,213
				2014
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2015	mov QWORD[rdi],r8
				2016	mov QWORD[8+rdi],r9
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2017	DB 102,73,15,126,217
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2018	mov QWORD[16+rdi],r10
				2019	mov QWORD[24+rdi],r11
				2020	mov QWORD[32+rdi],r12
				2021	mov QWORD[40+rdi],r13
				2022	mov QWORD[48+rdi],r14
				2023	mov QWORD[56+rdi],r15
				2024	lea rdi,[64+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2025
				2026	cmp rdi,rdx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2027	jb NEAR $L$8x_reduction_loop
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2028	DB 0F3h,0C3h ;repret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2029
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2030
				2031	ALIGN 32
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2032	__bn_post4x_internal:
				2033	mov r12,QWORD[rbp]
				2034	lea rbx,[r9*1+rdi]
				2035	mov rcx,r9
				2036	DB 102,72,15,126,207
				2037	neg rax
				2038	DB 102,72,15,126,206
				2039	sar rcx,3+2
				2040	dec r12
				2041	xor r10,r10
				2042	mov r13,QWORD[8+rbp]
				2043	mov r14,QWORD[16+rbp]
				2044	mov r15,QWORD[24+rbp]
				2045	jmp NEAR $L$sqr4x_sub_entry
				2046
				2047	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2048	$L$sqr4x_sub:
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2049	mov r12,QWORD[rbp]
				2050	mov r13,QWORD[8+rbp]
				2051	mov r14,QWORD[16+rbp]
				2052	mov r15,QWORD[24+rbp]
				2053	$L$sqr4x_sub_entry:
				2054	lea rbp,[32+rbp]
				2055	not r12
				2056	not r13
				2057	not r14
				2058	not r15
				2059	and r12,rax
				2060	and r13,rax
				2061	and r14,rax
				2062	and r15,rax
				2063
				2064	neg r10
				2065	adc r12,QWORD[rbx]
				2066	adc r13,QWORD[8+rbx]
				2067	adc r14,QWORD[16+rbx]
				2068	adc r15,QWORD[24+rbx]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2069	mov QWORD[rdi],r12
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2070	lea rbx,[32+rbx]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2071	mov QWORD[8+rdi],r13
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2072	sbb r10,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2073	mov QWORD[16+rdi],r14
				2074	mov QWORD[24+rdi],r15
				2075	lea rdi,[32+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2076
				2077	inc rcx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2078	jnz NEAR $L$sqr4x_sub
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2079
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2080	mov r10,r9
				2081	neg r9
				2082	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2083
				2084	global bn_from_montgomery
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2085
				2086	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2087	bn_from_montgomery:
				2088	test DWORD[48+rsp],7
				2089	jz NEAR bn_from_mont8x
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2090	xor eax,eax
				2091	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2092
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2093
				2094
				2095	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2096	bn_from_mont8x:
				2097	mov QWORD[8+rsp],rdi ;WIN64 prologue
				2098	mov QWORD[16+rsp],rsi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2099	mov rax,rsp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2100	$L$SEH_begin_bn_from_mont8x:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2101	mov rdi,rcx
				2102	mov rsi,rdx
				2103	mov rdx,r8
				2104	mov rcx,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2105	mov r8,QWORD[40+rsp]
				2106	mov r9,QWORD[48+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2107
				2108
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2109
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2110	DB 0x67
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2111	mov rax,rsp
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2112
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2113	push rbx
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2114
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2115	push rbp
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2116
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2117	push r12
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2118
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2119	push r13
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2120
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2121	push r14
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2122
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2123	push r15
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2124
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2125	$L$from_prologue:
				2126
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2127	shl r9d,3
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2128	lea r10,[r9*2+r9]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2129	neg r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2130	mov r8,QWORD[r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2131
				2132
				2133
				2134
				2135
				2136
				2137
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2138
				2139	lea r11,[((-320))+r9*2+rsp]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2140	mov rbp,rsp
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2141	sub r11,rdi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2142	and r11,4095
				2143	cmp r10,r11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2144	jb NEAR $L$from_sp_alt
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2145	sub rbp,r11
				2146	lea rbp,[((-320))+r9*2+rbp]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2147	jmp NEAR $L$from_sp_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2148
				2149	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2150	$L$from_sp_alt:
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2151	lea r10,[((4096-320))+r9*2]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2152	lea rbp,[((-320))+r9*2+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2153	sub r11,r10
				2154	mov r10,0
				2155	cmovc r11,r10
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2156	sub rbp,r11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2157	$L$from_sp_done:
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2158	and rbp,-64
				2159	mov r11,rsp
				2160	sub r11,rbp
				2161	and r11,-4096
				2162	lea rsp,[rbp*1+r11]
				2163	mov r10,QWORD[rsp]
				2164	cmp rsp,rbp
				2165	ja NEAR $L$from_page_walk
				2166	jmp NEAR $L$from_page_walk_done
				2167
				2168	$L$from_page_walk:
				2169	lea rsp,[((-4096))+rsp]
				2170	mov r10,QWORD[rsp]
				2171	cmp rsp,rbp
				2172	ja NEAR $L$from_page_walk
				2173	$L$from_page_walk_done:
				2174
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2175	mov r10,r9
				2176	neg r9
				2177
				2178
				2179
				2180
				2181
				2182
				2183
				2184
				2185
				2186
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2187	mov QWORD[32+rsp],r8
				2188	mov QWORD[40+rsp],rax
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2189
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2190	$L$from_body:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2191	mov r11,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2192	lea rax,[48+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2193	pxor xmm0,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2194	jmp NEAR $L$mul_by_1
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2195
				2196	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2197	$L$mul_by_1:
				2198	movdqu xmm1,XMMWORD[rsi]
				2199	movdqu xmm2,XMMWORD[16+rsi]
				2200	movdqu xmm3,XMMWORD[32+rsi]
				2201	movdqa XMMWORD[r9*1+rax],xmm0
				2202	movdqu xmm4,XMMWORD[48+rsi]
				2203	movdqa XMMWORD[16+r9*1+rax],xmm0
				2204	DB 0x48,0x8d,0xb6,0x40,0x00,0x00,0x00
				2205	movdqa XMMWORD[rax],xmm1
				2206	movdqa XMMWORD[32+r9*1+rax],xmm0
				2207	movdqa XMMWORD[16+rax],xmm2
				2208	movdqa XMMWORD[48+r9*1+rax],xmm0
				2209	movdqa XMMWORD[32+rax],xmm3
				2210	movdqa XMMWORD[48+rax],xmm4
				2211	lea rax,[64+rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2212	sub r11,64
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2213	jnz NEAR $L$mul_by_1
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2214
				2215	DB 102,72,15,110,207
				2216	DB 102,72,15,110,209
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2217	DB 0x67
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2218	mov rbp,rcx
				2219	DB 102,73,15,110,218
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2220	call __bn_sqr8x_reduction
				2221	call __bn_post4x_internal
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2222
				2223	pxor xmm0,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2224	lea rax,[48+rsp]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2225	jmp NEAR $L$from_mont_zero
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2226
				2227	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2228	$L$from_mont_zero:
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2229	mov rsi,QWORD[40+rsp]
				2230
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2231	movdqa XMMWORD[rax],xmm0
				2232	movdqa XMMWORD[16+rax],xmm0
				2233	movdqa XMMWORD[32+rax],xmm0
				2234	movdqa XMMWORD[48+rax],xmm0
				2235	lea rax,[64+rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2236	sub r9,32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2237	jnz NEAR $L$from_mont_zero
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2238
				2239	mov rax,1
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2240	mov r15,QWORD[((-48))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2241
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2242	mov r14,QWORD[((-40))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2243
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2244	mov r13,QWORD[((-32))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2245
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2246	mov r12,QWORD[((-24))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2247
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2248	mov rbp,QWORD[((-16))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2249
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2250	mov rbx,QWORD[((-8))+rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2251
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2252	lea rsp,[rsi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2253
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2254	$L$from_epilogue:
				2255	mov rdi,QWORD[8+rsp] ;WIN64 epilogue
				2256	mov rsi,QWORD[16+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2257	DB 0F3h,0C3h ;repret
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2258
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2259	$L$SEH_end_bn_from_mont8x:
				2260	global bn_scatter5
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2261
				2262	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2263	bn_scatter5:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2264	cmp edx,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2265	jz NEAR $L$scatter_epilogue
				2266	lea r8,[r9*8+r8]
				2267	$L$scatter:
				2268	mov rax,QWORD[rcx]
				2269	lea rcx,[8+rcx]
				2270	mov QWORD[r8],rax
				2271	lea r8,[256+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2272	sub edx,1
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2273	jnz NEAR $L$scatter
				2274	$L$scatter_epilogue:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2275	DB 0F3h,0C3h ;repret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2276
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2277
				2278	global bn_gather5
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2279
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2280	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2281	bn_gather5:
				2282	$L$SEH_begin_bn_gather5:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2283
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2284	DB 0x4c,0x8d,0x14,0x24
				2285	DB 0x48,0x81,0xec,0x08,0x01,0x00,0x00
				2286	lea rax,[$L$inc]
				2287	and rsp,-16
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2288
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2289	movd xmm5,r9d
				2290	movdqa xmm0,XMMWORD[rax]
				2291	movdqa xmm1,XMMWORD[16+rax]
				2292	lea r11,[128+r8]
				2293	lea rax,[128+rsp]
				2294
				2295	pshufd xmm5,xmm5,0
				2296	movdqa xmm4,xmm1
				2297	movdqa xmm2,xmm1
				2298	paddd xmm1,xmm0
				2299	pcmpeqd xmm0,xmm5
				2300	movdqa xmm3,xmm4
				2301
				2302	paddd xmm2,xmm1
				2303	pcmpeqd xmm1,xmm5
				2304	movdqa XMMWORD[(-128)+rax],xmm0
				2305	movdqa xmm0,xmm4
				2306
				2307	paddd xmm3,xmm2
				2308	pcmpeqd xmm2,xmm5
				2309	movdqa XMMWORD[(-112)+rax],xmm1
				2310	movdqa xmm1,xmm4
				2311
				2312	paddd xmm0,xmm3
				2313	pcmpeqd xmm3,xmm5
				2314	movdqa XMMWORD[(-96)+rax],xmm2
				2315	movdqa xmm2,xmm4
				2316	paddd xmm1,xmm0
				2317	pcmpeqd xmm0,xmm5
				2318	movdqa XMMWORD[(-80)+rax],xmm3
				2319	movdqa xmm3,xmm4
				2320
				2321	paddd xmm2,xmm1
				2322	pcmpeqd xmm1,xmm5
				2323	movdqa XMMWORD[(-64)+rax],xmm0
				2324	movdqa xmm0,xmm4
				2325
				2326	paddd xmm3,xmm2
				2327	pcmpeqd xmm2,xmm5
				2328	movdqa XMMWORD[(-48)+rax],xmm1
				2329	movdqa xmm1,xmm4
				2330
				2331	paddd xmm0,xmm3
				2332	pcmpeqd xmm3,xmm5
				2333	movdqa XMMWORD[(-32)+rax],xmm2
				2334	movdqa xmm2,xmm4
				2335	paddd xmm1,xmm0
				2336	pcmpeqd xmm0,xmm5
				2337	movdqa XMMWORD[(-16)+rax],xmm3
				2338	movdqa xmm3,xmm4
				2339
				2340	paddd xmm2,xmm1
				2341	pcmpeqd xmm1,xmm5
				2342	movdqa XMMWORD[rax],xmm0
				2343	movdqa xmm0,xmm4
				2344
				2345	paddd xmm3,xmm2
				2346	pcmpeqd xmm2,xmm5
				2347	movdqa XMMWORD[16+rax],xmm1
				2348	movdqa xmm1,xmm4
				2349
				2350	paddd xmm0,xmm3
				2351	pcmpeqd xmm3,xmm5
				2352	movdqa XMMWORD[32+rax],xmm2
				2353	movdqa xmm2,xmm4
				2354	paddd xmm1,xmm0
				2355	pcmpeqd xmm0,xmm5
				2356	movdqa XMMWORD[48+rax],xmm3
				2357	movdqa xmm3,xmm4
				2358
				2359	paddd xmm2,xmm1
				2360	pcmpeqd xmm1,xmm5
				2361	movdqa XMMWORD[64+rax],xmm0
				2362	movdqa xmm0,xmm4
				2363
				2364	paddd xmm3,xmm2
				2365	pcmpeqd xmm2,xmm5
				2366	movdqa XMMWORD[80+rax],xmm1
				2367	movdqa xmm1,xmm4
				2368
				2369	paddd xmm0,xmm3
				2370	pcmpeqd xmm3,xmm5
				2371	movdqa XMMWORD[96+rax],xmm2
				2372	movdqa xmm2,xmm4
				2373	movdqa XMMWORD[112+rax],xmm3
				2374	jmp NEAR $L$gather
				2375
				2376	ALIGN 32
				2377	$L$gather:
				2378	pxor xmm4,xmm4
				2379	pxor xmm5,xmm5
				2380	movdqa xmm0,XMMWORD[((-128))+r11]
				2381	movdqa xmm1,XMMWORD[((-112))+r11]
				2382	movdqa xmm2,XMMWORD[((-96))+r11]
				2383	pand xmm0,XMMWORD[((-128))+rax]
				2384	movdqa xmm3,XMMWORD[((-80))+r11]
				2385	pand xmm1,XMMWORD[((-112))+rax]
				2386	por xmm4,xmm0
				2387	pand xmm2,XMMWORD[((-96))+rax]
				2388	por xmm5,xmm1
				2389	pand xmm3,XMMWORD[((-80))+rax]
				2390	por xmm4,xmm2
				2391	por xmm5,xmm3
				2392	movdqa xmm0,XMMWORD[((-64))+r11]
				2393	movdqa xmm1,XMMWORD[((-48))+r11]
				2394	movdqa xmm2,XMMWORD[((-32))+r11]
				2395	pand xmm0,XMMWORD[((-64))+rax]
				2396	movdqa xmm3,XMMWORD[((-16))+r11]
				2397	pand xmm1,XMMWORD[((-48))+rax]
				2398	por xmm4,xmm0
				2399	pand xmm2,XMMWORD[((-32))+rax]
				2400	por xmm5,xmm1
				2401	pand xmm3,XMMWORD[((-16))+rax]
				2402	por xmm4,xmm2
				2403	por xmm5,xmm3
				2404	movdqa xmm0,XMMWORD[r11]
				2405	movdqa xmm1,XMMWORD[16+r11]
				2406	movdqa xmm2,XMMWORD[32+r11]
				2407	pand xmm0,XMMWORD[rax]
				2408	movdqa xmm3,XMMWORD[48+r11]
				2409	pand xmm1,XMMWORD[16+rax]
				2410	por xmm4,xmm0
				2411	pand xmm2,XMMWORD[32+rax]
				2412	por xmm5,xmm1
				2413	pand xmm3,XMMWORD[48+rax]
				2414	por xmm4,xmm2
				2415	por xmm5,xmm3
				2416	movdqa xmm0,XMMWORD[64+r11]
				2417	movdqa xmm1,XMMWORD[80+r11]
				2418	movdqa xmm2,XMMWORD[96+r11]
				2419	pand xmm0,XMMWORD[64+rax]
				2420	movdqa xmm3,XMMWORD[112+r11]
				2421	pand xmm1,XMMWORD[80+rax]
				2422	por xmm4,xmm0
				2423	pand xmm2,XMMWORD[96+rax]
				2424	por xmm5,xmm1
				2425	pand xmm3,XMMWORD[112+rax]
				2426	por xmm4,xmm2
				2427	por xmm5,xmm3
				2428	por xmm4,xmm5
				2429	lea r11,[256+r11]
				2430	pshufd xmm0,xmm4,0x4e
				2431	por xmm0,xmm4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2432	movq QWORD[rcx],xmm0
				2433	lea rcx,[8+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2434	sub edx,1
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2435	jnz NEAR $L$gather
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2436
				2437	lea rsp,[r10]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2438	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2439	$L$SEH_end_bn_gather5:
				2440
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2441	ALIGN 64
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2442	$L$inc:
				2443	DD 0,0,1,1
				2444	DD 2,2,2,2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2445	DB 77,111,110,116,103,111,109,101,114,121,32,77,117,108,116,105
				2446	DB 112,108,105,99,97,116,105,111,110,32,119,105,116,104,32,115
				2447	DB 99,97,116,116,101,114,47,103,97,116,104,101,114,32,102,111
				2448	DB 114,32,120,56,54,95,54,52,44,32,67,82,89,80,84,79
				2449	DB 71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111
				2450	DB 112,101,110,115,115,108,46,111,114,103,62,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2451	EXTERN __imp_RtlVirtualUnwind
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2452
				2453	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2454	mul_handler:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2455	push rsi
				2456	push rdi
				2457	push rbx
				2458	push rbp
				2459	push r12
				2460	push r13
				2461	push r14
				2462	push r15
				2463	pushfq
				2464	sub rsp,64
				2465
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2466	mov rax,QWORD[120+r8]
				2467	mov rbx,QWORD[248+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2468
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2469	mov rsi,QWORD[8+r9]
				2470	mov r11,QWORD[56+r9]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2471
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2472	mov r10d,DWORD[r11]
				2473	lea r10,[r10*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2474	cmp rbx,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2475	jb NEAR $L$common_seh_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2476
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2477	mov r10d,DWORD[4+r11]
				2478	lea r10,[r10*1+rsi]
				2479	cmp rbx,r10
				2480	jb NEAR $L$common_pop_regs
				2481
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2482	mov rax,QWORD[152+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2483
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2484	mov r10d,DWORD[8+r11]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2485	lea r10,[r10*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2486	cmp rbx,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2487	jae NEAR $L$common_seh_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2488
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2489	lea r10,[$L$mul_epilogue]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2490	cmp rbx,r10
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2491	ja NEAR $L$body_40
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2492
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2493	mov r10,QWORD[192+r8]
				2494	mov rax,QWORD[8+r10*8+rax]
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2495
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2496	jmp NEAR $L$common_pop_regs
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2497
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2498	$L$body_40:
				2499	mov rax,QWORD[40+rax]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2500	$L$common_pop_regs:
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2501	mov rbx,QWORD[((-8))+rax]
				2502	mov rbp,QWORD[((-16))+rax]
				2503	mov r12,QWORD[((-24))+rax]
				2504	mov r13,QWORD[((-32))+rax]
				2505	mov r14,QWORD[((-40))+rax]
				2506	mov r15,QWORD[((-48))+rax]
				2507	mov QWORD[144+r8],rbx
				2508	mov QWORD[160+r8],rbp
				2509	mov QWORD[216+r8],r12
				2510	mov QWORD[224+r8],r13
				2511	mov QWORD[232+r8],r14
				2512	mov QWORD[240+r8],r15
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2513
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2514	$L$common_seh_tail:
				2515	mov rdi,QWORD[8+rax]
				2516	mov rsi,QWORD[16+rax]
				2517	mov QWORD[152+r8],rax
				2518	mov QWORD[168+r8],rsi
				2519	mov QWORD[176+r8],rdi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2520
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2521	mov rdi,QWORD[40+r9]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2522	mov rsi,r8
				2523	mov ecx,154
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2524	DD 0xa548f3fc
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2525
				2526	mov rsi,r9
				2527	xor rcx,rcx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2528	mov rdx,QWORD[8+rsi]
				2529	mov r8,QWORD[rsi]
				2530	mov r9,QWORD[16+rsi]
				2531	mov r10,QWORD[40+rsi]
				2532	lea r11,[56+rsi]
				2533	lea r12,[24+rsi]
				2534	mov QWORD[32+rsp],r10
				2535	mov QWORD[40+rsp],r11
				2536	mov QWORD[48+rsp],r12
				2537	mov QWORD[56+rsp],rcx
				2538	call QWORD[__imp_RtlVirtualUnwind]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2539
				2540	mov eax,1
				2541	add rsp,64
				2542	popfq
				2543	pop r15
				2544	pop r14
				2545	pop r13
				2546	pop r12
				2547	pop rbp
				2548	pop rbx
				2549	pop rdi
				2550	pop rsi
				2551	DB 0F3h,0C3h ;repret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2552
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2553
				2554	section .pdata rdata align=4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2555	ALIGN 4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2556	DD $L$SEH_begin_bn_mul_mont_gather5 wrt ..imagebase
				2557	DD $L$SEH_end_bn_mul_mont_gather5 wrt ..imagebase
				2558	DD $L$SEH_info_bn_mul_mont_gather5 wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2559
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2560	DD $L$SEH_begin_bn_mul4x_mont_gather5 wrt ..imagebase
				2561	DD $L$SEH_end_bn_mul4x_mont_gather5 wrt ..imagebase
				2562	DD $L$SEH_info_bn_mul4x_mont_gather5 wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2563
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2564	DD $L$SEH_begin_bn_power5 wrt ..imagebase
				2565	DD $L$SEH_end_bn_power5 wrt ..imagebase
				2566	DD $L$SEH_info_bn_power5 wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2567
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2568	DD $L$SEH_begin_bn_from_mont8x wrt ..imagebase
				2569	DD $L$SEH_end_bn_from_mont8x wrt ..imagebase
				2570	DD $L$SEH_info_bn_from_mont8x wrt ..imagebase
				2571	DD $L$SEH_begin_bn_gather5 wrt ..imagebase
				2572	DD $L$SEH_end_bn_gather5 wrt ..imagebase
				2573	DD $L$SEH_info_bn_gather5 wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2574
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2575	section .xdata rdata align=8
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2576	ALIGN 8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2577	$L$SEH_info_bn_mul_mont_gather5:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2578	DB 9,0,0,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2579	DD mul_handler wrt ..imagebase
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2580	DD $L$mul_body wrt ..imagebase,$L$mul_body wrt ..imagebase,$L$mul_epilogue wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2581	ALIGN 8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2582	$L$SEH_info_bn_mul4x_mont_gather5:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2583	DB 9,0,0,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2584	DD mul_handler wrt ..imagebase
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2585	DD $L$mul4x_prologue wrt ..imagebase,$L$mul4x_body wrt ..imagebase,$L$mul4x_epilogue wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2586	ALIGN 8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2587	$L$SEH_info_bn_power5:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2588	DB 9,0,0,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2589	DD mul_handler wrt ..imagebase
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2590	DD $L$power5_prologue wrt ..imagebase,$L$power5_body wrt ..imagebase,$L$power5_epilogue wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2591	ALIGN 8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2592	$L$SEH_info_bn_from_mont8x:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2593	DB 9,0,0,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2594	DD mul_handler wrt ..imagebase
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2595	DD $L$from_prologue wrt ..imagebase,$L$from_body wrt ..imagebase,$L$from_epilogue wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2596	ALIGN 8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2597	$L$SEH_info_bn_gather5:
David Benjamin	4969cc9	2016-04-22 15:02:23 -0400	[diff] [blame]	2598	DB 0x01,0x0b,0x03,0x0a
				2599	DB 0x0b,0x01,0x21,0x00
				2600	DB 0x04,0xa3,0x00,0x00
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2601	ALIGN 8