Blame - win-x86_64/crypto/fipsmodule/aesni-x86_64.asm - platform/external/boringssl

blob: 13e9c5e5b6b16090decb2a85a213f1029e1e057e [file] [log] [blame]

Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1	default rel
				2	%define XMMWORD
				3	%define YMMWORD
				4	%define ZMMWORD
				5	section .text code align=64
				6
Robert Sloan	2424d84	2017-05-01 07:46:28 -0700	[diff] [blame]	7	EXTERN OPENSSL_ia32cap_P
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	8	global aesni_encrypt
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	9
				10	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	11	aesni_encrypt:
				12	movups xmm2,XMMWORD[rcx]
				13	mov eax,DWORD[240+r8]
				14	movups xmm0,XMMWORD[r8]
				15	movups xmm1,XMMWORD[16+r8]
				16	lea r8,[32+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	17	xorps xmm2,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	18	$L$oop_enc1_1:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	19	DB 102,15,56,220,209
				20	dec eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	21	movups xmm1,XMMWORD[r8]
				22	lea r8,[16+r8]
				23	jnz NEAR $L$oop_enc1_1
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	24	DB 102,15,56,221,209
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	25	pxor xmm0,xmm0
				26	pxor xmm1,xmm1
				27	movups XMMWORD[rdx],xmm2
				28	pxor xmm2,xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	29	DB 0F3h,0C3h ;repret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	30
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	31
				32	global aesni_decrypt
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	33
				34	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	35	aesni_decrypt:
				36	movups xmm2,XMMWORD[rcx]
				37	mov eax,DWORD[240+r8]
				38	movups xmm0,XMMWORD[r8]
				39	movups xmm1,XMMWORD[16+r8]
				40	lea r8,[32+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	41	xorps xmm2,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	42	$L$oop_dec1_2:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	43	DB 102,15,56,222,209
				44	dec eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	45	movups xmm1,XMMWORD[r8]
				46	lea r8,[16+r8]
				47	jnz NEAR $L$oop_dec1_2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	48	DB 102,15,56,223,209
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	49	pxor xmm0,xmm0
				50	pxor xmm1,xmm1
				51	movups XMMWORD[rdx],xmm2
				52	pxor xmm2,xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	53	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	54
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	55
				56	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	57	_aesni_encrypt2:
				58	movups xmm0,XMMWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	59	shl eax,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	60	movups xmm1,XMMWORD[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	61	xorps xmm2,xmm0
				62	xorps xmm3,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	63	movups xmm0,XMMWORD[32+rcx]
				64	lea rcx,[32+rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	65	neg rax
				66	add rax,16
				67
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	68	$L$enc_loop2:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	69	DB 102,15,56,220,209
				70	DB 102,15,56,220,217
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	71	movups xmm1,XMMWORD[rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	72	add rax,32
				73	DB 102,15,56,220,208
				74	DB 102,15,56,220,216
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	75	movups xmm0,XMMWORD[((-16))+rax*1+rcx]
				76	jnz NEAR $L$enc_loop2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	77
				78	DB 102,15,56,220,209
				79	DB 102,15,56,220,217
				80	DB 102,15,56,221,208
				81	DB 102,15,56,221,216
				82	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	83
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	84
				85	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	86	_aesni_decrypt2:
				87	movups xmm0,XMMWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	88	shl eax,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	89	movups xmm1,XMMWORD[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	90	xorps xmm2,xmm0
				91	xorps xmm3,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	92	movups xmm0,XMMWORD[32+rcx]
				93	lea rcx,[32+rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	94	neg rax
				95	add rax,16
				96
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	97	$L$dec_loop2:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	98	DB 102,15,56,222,209
				99	DB 102,15,56,222,217
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	100	movups xmm1,XMMWORD[rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	101	add rax,32
				102	DB 102,15,56,222,208
				103	DB 102,15,56,222,216
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	104	movups xmm0,XMMWORD[((-16))+rax*1+rcx]
				105	jnz NEAR $L$dec_loop2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	106
				107	DB 102,15,56,222,209
				108	DB 102,15,56,222,217
				109	DB 102,15,56,223,208
				110	DB 102,15,56,223,216
				111	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	112
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	113
				114	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	115	_aesni_encrypt3:
				116	movups xmm0,XMMWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	117	shl eax,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	118	movups xmm1,XMMWORD[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	119	xorps xmm2,xmm0
				120	xorps xmm3,xmm0
				121	xorps xmm4,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	122	movups xmm0,XMMWORD[32+rcx]
				123	lea rcx,[32+rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	124	neg rax
				125	add rax,16
				126
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	127	$L$enc_loop3:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	128	DB 102,15,56,220,209
				129	DB 102,15,56,220,217
				130	DB 102,15,56,220,225
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	131	movups xmm1,XMMWORD[rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	132	add rax,32
				133	DB 102,15,56,220,208
				134	DB 102,15,56,220,216
				135	DB 102,15,56,220,224
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	136	movups xmm0,XMMWORD[((-16))+rax*1+rcx]
				137	jnz NEAR $L$enc_loop3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	138
				139	DB 102,15,56,220,209
				140	DB 102,15,56,220,217
				141	DB 102,15,56,220,225
				142	DB 102,15,56,221,208
				143	DB 102,15,56,221,216
				144	DB 102,15,56,221,224
				145	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	146
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	147
				148	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	149	_aesni_decrypt3:
				150	movups xmm0,XMMWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	151	shl eax,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	152	movups xmm1,XMMWORD[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	153	xorps xmm2,xmm0
				154	xorps xmm3,xmm0
				155	xorps xmm4,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	156	movups xmm0,XMMWORD[32+rcx]
				157	lea rcx,[32+rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	158	neg rax
				159	add rax,16
				160
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	161	$L$dec_loop3:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	162	DB 102,15,56,222,209
				163	DB 102,15,56,222,217
				164	DB 102,15,56,222,225
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	165	movups xmm1,XMMWORD[rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	166	add rax,32
				167	DB 102,15,56,222,208
				168	DB 102,15,56,222,216
				169	DB 102,15,56,222,224
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	170	movups xmm0,XMMWORD[((-16))+rax*1+rcx]
				171	jnz NEAR $L$dec_loop3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	172
				173	DB 102,15,56,222,209
				174	DB 102,15,56,222,217
				175	DB 102,15,56,222,225
				176	DB 102,15,56,223,208
				177	DB 102,15,56,223,216
				178	DB 102,15,56,223,224
				179	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	180
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	181
				182	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	183	_aesni_encrypt4:
				184	movups xmm0,XMMWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	185	shl eax,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	186	movups xmm1,XMMWORD[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	187	xorps xmm2,xmm0
				188	xorps xmm3,xmm0
				189	xorps xmm4,xmm0
				190	xorps xmm5,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	191	movups xmm0,XMMWORD[32+rcx]
				192	lea rcx,[32+rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	193	neg rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	194	DB 0x0f,0x1f,0x00
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	195	add rax,16
				196
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	197	$L$enc_loop4:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	198	DB 102,15,56,220,209
				199	DB 102,15,56,220,217
				200	DB 102,15,56,220,225
				201	DB 102,15,56,220,233
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	202	movups xmm1,XMMWORD[rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	203	add rax,32
				204	DB 102,15,56,220,208
				205	DB 102,15,56,220,216
				206	DB 102,15,56,220,224
				207	DB 102,15,56,220,232
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	208	movups xmm0,XMMWORD[((-16))+rax*1+rcx]
				209	jnz NEAR $L$enc_loop4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	210
				211	DB 102,15,56,220,209
				212	DB 102,15,56,220,217
				213	DB 102,15,56,220,225
				214	DB 102,15,56,220,233
				215	DB 102,15,56,221,208
				216	DB 102,15,56,221,216
				217	DB 102,15,56,221,224
				218	DB 102,15,56,221,232
				219	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	220
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	221
				222	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	223	_aesni_decrypt4:
				224	movups xmm0,XMMWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	225	shl eax,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	226	movups xmm1,XMMWORD[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	227	xorps xmm2,xmm0
				228	xorps xmm3,xmm0
				229	xorps xmm4,xmm0
				230	xorps xmm5,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	231	movups xmm0,XMMWORD[32+rcx]
				232	lea rcx,[32+rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	233	neg rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	234	DB 0x0f,0x1f,0x00
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	235	add rax,16
				236
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	237	$L$dec_loop4:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	238	DB 102,15,56,222,209
				239	DB 102,15,56,222,217
				240	DB 102,15,56,222,225
				241	DB 102,15,56,222,233
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	242	movups xmm1,XMMWORD[rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	243	add rax,32
				244	DB 102,15,56,222,208
				245	DB 102,15,56,222,216
				246	DB 102,15,56,222,224
				247	DB 102,15,56,222,232
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	248	movups xmm0,XMMWORD[((-16))+rax*1+rcx]
				249	jnz NEAR $L$dec_loop4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	250
				251	DB 102,15,56,222,209
				252	DB 102,15,56,222,217
				253	DB 102,15,56,222,225
				254	DB 102,15,56,222,233
				255	DB 102,15,56,223,208
				256	DB 102,15,56,223,216
				257	DB 102,15,56,223,224
				258	DB 102,15,56,223,232
				259	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	260
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	261
				262	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	263	_aesni_encrypt6:
				264	movups xmm0,XMMWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	265	shl eax,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	266	movups xmm1,XMMWORD[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	267	xorps xmm2,xmm0
				268	pxor xmm3,xmm0
				269	pxor xmm4,xmm0
				270	DB 102,15,56,220,209
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	271	lea rcx,[32+rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	272	neg rax
				273	DB 102,15,56,220,217
				274	pxor xmm5,xmm0
				275	pxor xmm6,xmm0
				276	DB 102,15,56,220,225
				277	pxor xmm7,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	278	movups xmm0,XMMWORD[rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	279	add rax,16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	280	jmp NEAR $L$enc_loop6_enter
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	281	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	282	$L$enc_loop6:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	283	DB 102,15,56,220,209
				284	DB 102,15,56,220,217
				285	DB 102,15,56,220,225
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	286	$L$enc_loop6_enter:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	287	DB 102,15,56,220,233
				288	DB 102,15,56,220,241
				289	DB 102,15,56,220,249
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	290	movups xmm1,XMMWORD[rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	291	add rax,32
				292	DB 102,15,56,220,208
				293	DB 102,15,56,220,216
				294	DB 102,15,56,220,224
				295	DB 102,15,56,220,232
				296	DB 102,15,56,220,240
				297	DB 102,15,56,220,248
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	298	movups xmm0,XMMWORD[((-16))+rax*1+rcx]
				299	jnz NEAR $L$enc_loop6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	300
				301	DB 102,15,56,220,209
				302	DB 102,15,56,220,217
				303	DB 102,15,56,220,225
				304	DB 102,15,56,220,233
				305	DB 102,15,56,220,241
				306	DB 102,15,56,220,249
				307	DB 102,15,56,221,208
				308	DB 102,15,56,221,216
				309	DB 102,15,56,221,224
				310	DB 102,15,56,221,232
				311	DB 102,15,56,221,240
				312	DB 102,15,56,221,248
				313	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	314
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	315
				316	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	317	_aesni_decrypt6:
				318	movups xmm0,XMMWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	319	shl eax,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	320	movups xmm1,XMMWORD[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	321	xorps xmm2,xmm0
				322	pxor xmm3,xmm0
				323	pxor xmm4,xmm0
				324	DB 102,15,56,222,209
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	325	lea rcx,[32+rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	326	neg rax
				327	DB 102,15,56,222,217
				328	pxor xmm5,xmm0
				329	pxor xmm6,xmm0
				330	DB 102,15,56,222,225
				331	pxor xmm7,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	332	movups xmm0,XMMWORD[rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	333	add rax,16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	334	jmp NEAR $L$dec_loop6_enter
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	335	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	336	$L$dec_loop6:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	337	DB 102,15,56,222,209
				338	DB 102,15,56,222,217
				339	DB 102,15,56,222,225
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	340	$L$dec_loop6_enter:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	341	DB 102,15,56,222,233
				342	DB 102,15,56,222,241
				343	DB 102,15,56,222,249
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	344	movups xmm1,XMMWORD[rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	345	add rax,32
				346	DB 102,15,56,222,208
				347	DB 102,15,56,222,216
				348	DB 102,15,56,222,224
				349	DB 102,15,56,222,232
				350	DB 102,15,56,222,240
				351	DB 102,15,56,222,248
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	352	movups xmm0,XMMWORD[((-16))+rax*1+rcx]
				353	jnz NEAR $L$dec_loop6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	354
				355	DB 102,15,56,222,209
				356	DB 102,15,56,222,217
				357	DB 102,15,56,222,225
				358	DB 102,15,56,222,233
				359	DB 102,15,56,222,241
				360	DB 102,15,56,222,249
				361	DB 102,15,56,223,208
				362	DB 102,15,56,223,216
				363	DB 102,15,56,223,224
				364	DB 102,15,56,223,232
				365	DB 102,15,56,223,240
				366	DB 102,15,56,223,248
				367	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	368
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	369
				370	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	371	_aesni_encrypt8:
				372	movups xmm0,XMMWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	373	shl eax,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	374	movups xmm1,XMMWORD[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	375	xorps xmm2,xmm0
				376	xorps xmm3,xmm0
				377	pxor xmm4,xmm0
				378	pxor xmm5,xmm0
				379	pxor xmm6,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	380	lea rcx,[32+rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	381	neg rax
				382	DB 102,15,56,220,209
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	383	pxor xmm7,xmm0
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	384	pxor xmm8,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	385	DB 102,15,56,220,217
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	386	pxor xmm9,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	387	movups xmm0,XMMWORD[rax*1+rcx]
				388	add rax,16
				389	jmp NEAR $L$enc_loop8_inner
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	390	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	391	$L$enc_loop8:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	392	DB 102,15,56,220,209
				393	DB 102,15,56,220,217
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	394	$L$enc_loop8_inner:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	395	DB 102,15,56,220,225
				396	DB 102,15,56,220,233
				397	DB 102,15,56,220,241
				398	DB 102,15,56,220,249
				399	DB 102,68,15,56,220,193
				400	DB 102,68,15,56,220,201
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	401	$L$enc_loop8_enter:
				402	movups xmm1,XMMWORD[rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	403	add rax,32
				404	DB 102,15,56,220,208
				405	DB 102,15,56,220,216
				406	DB 102,15,56,220,224
				407	DB 102,15,56,220,232
				408	DB 102,15,56,220,240
				409	DB 102,15,56,220,248
				410	DB 102,68,15,56,220,192
				411	DB 102,68,15,56,220,200
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	412	movups xmm0,XMMWORD[((-16))+rax*1+rcx]
				413	jnz NEAR $L$enc_loop8
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	414
				415	DB 102,15,56,220,209
				416	DB 102,15,56,220,217
				417	DB 102,15,56,220,225
				418	DB 102,15,56,220,233
				419	DB 102,15,56,220,241
				420	DB 102,15,56,220,249
				421	DB 102,68,15,56,220,193
				422	DB 102,68,15,56,220,201
				423	DB 102,15,56,221,208
				424	DB 102,15,56,221,216
				425	DB 102,15,56,221,224
				426	DB 102,15,56,221,232
				427	DB 102,15,56,221,240
				428	DB 102,15,56,221,248
				429	DB 102,68,15,56,221,192
				430	DB 102,68,15,56,221,200
				431	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	432
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	433
				434	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	435	_aesni_decrypt8:
				436	movups xmm0,XMMWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	437	shl eax,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	438	movups xmm1,XMMWORD[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	439	xorps xmm2,xmm0
				440	xorps xmm3,xmm0
				441	pxor xmm4,xmm0
				442	pxor xmm5,xmm0
				443	pxor xmm6,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	444	lea rcx,[32+rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	445	neg rax
				446	DB 102,15,56,222,209
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	447	pxor xmm7,xmm0
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	448	pxor xmm8,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	449	DB 102,15,56,222,217
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	450	pxor xmm9,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	451	movups xmm0,XMMWORD[rax*1+rcx]
				452	add rax,16
				453	jmp NEAR $L$dec_loop8_inner
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	454	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	455	$L$dec_loop8:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	456	DB 102,15,56,222,209
				457	DB 102,15,56,222,217
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	458	$L$dec_loop8_inner:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	459	DB 102,15,56,222,225
				460	DB 102,15,56,222,233
				461	DB 102,15,56,222,241
				462	DB 102,15,56,222,249
				463	DB 102,68,15,56,222,193
				464	DB 102,68,15,56,222,201
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	465	$L$dec_loop8_enter:
				466	movups xmm1,XMMWORD[rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	467	add rax,32
				468	DB 102,15,56,222,208
				469	DB 102,15,56,222,216
				470	DB 102,15,56,222,224
				471	DB 102,15,56,222,232
				472	DB 102,15,56,222,240
				473	DB 102,15,56,222,248
				474	DB 102,68,15,56,222,192
				475	DB 102,68,15,56,222,200
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	476	movups xmm0,XMMWORD[((-16))+rax*1+rcx]
				477	jnz NEAR $L$dec_loop8
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	478
				479	DB 102,15,56,222,209
				480	DB 102,15,56,222,217
				481	DB 102,15,56,222,225
				482	DB 102,15,56,222,233
				483	DB 102,15,56,222,241
				484	DB 102,15,56,222,249
				485	DB 102,68,15,56,222,193
				486	DB 102,68,15,56,222,201
				487	DB 102,15,56,223,208
				488	DB 102,15,56,223,216
				489	DB 102,15,56,223,224
				490	DB 102,15,56,223,232
				491	DB 102,15,56,223,240
				492	DB 102,15,56,223,248
				493	DB 102,68,15,56,223,192
				494	DB 102,68,15,56,223,200
				495	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	496
				497	global aesni_ecb_encrypt
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	498
				499	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	500	aesni_ecb_encrypt:
				501	mov QWORD[8+rsp],rdi ;WIN64 prologue
				502	mov QWORD[16+rsp],rsi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	503	mov rax,rsp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	504	$L$SEH_begin_aesni_ecb_encrypt:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	505	mov rdi,rcx
				506	mov rsi,rdx
				507	mov rdx,r8
				508	mov rcx,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	509	mov r8,QWORD[40+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	510
				511
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	512	lea rsp,[((-88))+rsp]
				513	movaps XMMWORD[rsp],xmm6
				514	movaps XMMWORD[16+rsp],xmm7
				515	movaps XMMWORD[32+rsp],xmm8
				516	movaps XMMWORD[48+rsp],xmm9
				517	$L$ecb_enc_body:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	518	and rdx,-16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	519	jz NEAR $L$ecb_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	520
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	521	mov eax,DWORD[240+rcx]
				522	movups xmm0,XMMWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	523	mov r11,rcx
				524	mov r10d,eax
				525	test r8d,r8d
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	526	jz NEAR $L$ecb_decrypt
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	527
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	528	cmp rdx,0x80
				529	jb NEAR $L$ecb_enc_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	530
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	531	movdqu xmm2,XMMWORD[rdi]
				532	movdqu xmm3,XMMWORD[16+rdi]
				533	movdqu xmm4,XMMWORD[32+rdi]
				534	movdqu xmm5,XMMWORD[48+rdi]
				535	movdqu xmm6,XMMWORD[64+rdi]
				536	movdqu xmm7,XMMWORD[80+rdi]
				537	movdqu xmm8,XMMWORD[96+rdi]
				538	movdqu xmm9,XMMWORD[112+rdi]
				539	lea rdi,[128+rdi]
				540	sub rdx,0x80
				541	jmp NEAR $L$ecb_enc_loop8_enter
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	542	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	543	$L$ecb_enc_loop8:
				544	movups XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	545	mov rcx,r11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	546	movdqu xmm2,XMMWORD[rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	547	mov eax,r10d
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	548	movups XMMWORD[16+rsi],xmm3
				549	movdqu xmm3,XMMWORD[16+rdi]
				550	movups XMMWORD[32+rsi],xmm4
				551	movdqu xmm4,XMMWORD[32+rdi]
				552	movups XMMWORD[48+rsi],xmm5
				553	movdqu xmm5,XMMWORD[48+rdi]
				554	movups XMMWORD[64+rsi],xmm6
				555	movdqu xmm6,XMMWORD[64+rdi]
				556	movups XMMWORD[80+rsi],xmm7
				557	movdqu xmm7,XMMWORD[80+rdi]
				558	movups XMMWORD[96+rsi],xmm8
				559	movdqu xmm8,XMMWORD[96+rdi]
				560	movups XMMWORD[112+rsi],xmm9
				561	lea rsi,[128+rsi]
				562	movdqu xmm9,XMMWORD[112+rdi]
				563	lea rdi,[128+rdi]
				564	$L$ecb_enc_loop8_enter:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	565
				566	call _aesni_encrypt8
				567
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	568	sub rdx,0x80
				569	jnc NEAR $L$ecb_enc_loop8
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	570
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	571	movups XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	572	mov rcx,r11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	573	movups XMMWORD[16+rsi],xmm3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	574	mov eax,r10d
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	575	movups XMMWORD[32+rsi],xmm4
				576	movups XMMWORD[48+rsi],xmm5
				577	movups XMMWORD[64+rsi],xmm6
				578	movups XMMWORD[80+rsi],xmm7
				579	movups XMMWORD[96+rsi],xmm8
				580	movups XMMWORD[112+rsi],xmm9
				581	lea rsi,[128+rsi]
				582	add rdx,0x80
				583	jz NEAR $L$ecb_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	584
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	585	$L$ecb_enc_tail:
				586	movups xmm2,XMMWORD[rdi]
				587	cmp rdx,0x20
				588	jb NEAR $L$ecb_enc_one
				589	movups xmm3,XMMWORD[16+rdi]
				590	je NEAR $L$ecb_enc_two
				591	movups xmm4,XMMWORD[32+rdi]
				592	cmp rdx,0x40
				593	jb NEAR $L$ecb_enc_three
				594	movups xmm5,XMMWORD[48+rdi]
				595	je NEAR $L$ecb_enc_four
				596	movups xmm6,XMMWORD[64+rdi]
				597	cmp rdx,0x60
				598	jb NEAR $L$ecb_enc_five
				599	movups xmm7,XMMWORD[80+rdi]
				600	je NEAR $L$ecb_enc_six
				601	movdqu xmm8,XMMWORD[96+rdi]
				602	xorps xmm9,xmm9
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	603	call _aesni_encrypt8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	604	movups XMMWORD[rsi],xmm2
				605	movups XMMWORD[16+rsi],xmm3
				606	movups XMMWORD[32+rsi],xmm4
				607	movups XMMWORD[48+rsi],xmm5
				608	movups XMMWORD[64+rsi],xmm6
				609	movups XMMWORD[80+rsi],xmm7
				610	movups XMMWORD[96+rsi],xmm8
				611	jmp NEAR $L$ecb_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	612	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	613	$L$ecb_enc_one:
				614	movups xmm0,XMMWORD[rcx]
				615	movups xmm1,XMMWORD[16+rcx]
				616	lea rcx,[32+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	617	xorps xmm2,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	618	$L$oop_enc1_3:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	619	DB 102,15,56,220,209
				620	dec eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	621	movups xmm1,XMMWORD[rcx]
				622	lea rcx,[16+rcx]
				623	jnz NEAR $L$oop_enc1_3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	624	DB 102,15,56,221,209
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	625	movups XMMWORD[rsi],xmm2
				626	jmp NEAR $L$ecb_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	627	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	628	$L$ecb_enc_two:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	629	call _aesni_encrypt2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	630	movups XMMWORD[rsi],xmm2
				631	movups XMMWORD[16+rsi],xmm3
				632	jmp NEAR $L$ecb_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	633	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	634	$L$ecb_enc_three:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	635	call _aesni_encrypt3
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	636	movups XMMWORD[rsi],xmm2
				637	movups XMMWORD[16+rsi],xmm3
				638	movups XMMWORD[32+rsi],xmm4
				639	jmp NEAR $L$ecb_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	640	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	641	$L$ecb_enc_four:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	642	call _aesni_encrypt4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	643	movups XMMWORD[rsi],xmm2
				644	movups XMMWORD[16+rsi],xmm3
				645	movups XMMWORD[32+rsi],xmm4
				646	movups XMMWORD[48+rsi],xmm5
				647	jmp NEAR $L$ecb_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	648	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	649	$L$ecb_enc_five:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	650	xorps xmm7,xmm7
				651	call _aesni_encrypt6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	652	movups XMMWORD[rsi],xmm2
				653	movups XMMWORD[16+rsi],xmm3
				654	movups XMMWORD[32+rsi],xmm4
				655	movups XMMWORD[48+rsi],xmm5
				656	movups XMMWORD[64+rsi],xmm6
				657	jmp NEAR $L$ecb_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	658	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	659	$L$ecb_enc_six:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	660	call _aesni_encrypt6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	661	movups XMMWORD[rsi],xmm2
				662	movups XMMWORD[16+rsi],xmm3
				663	movups XMMWORD[32+rsi],xmm4
				664	movups XMMWORD[48+rsi],xmm5
				665	movups XMMWORD[64+rsi],xmm6
				666	movups XMMWORD[80+rsi],xmm7
				667	jmp NEAR $L$ecb_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	668
				669	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	670	$L$ecb_decrypt:
				671	cmp rdx,0x80
				672	jb NEAR $L$ecb_dec_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	673
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	674	movdqu xmm2,XMMWORD[rdi]
				675	movdqu xmm3,XMMWORD[16+rdi]
				676	movdqu xmm4,XMMWORD[32+rdi]
				677	movdqu xmm5,XMMWORD[48+rdi]
				678	movdqu xmm6,XMMWORD[64+rdi]
				679	movdqu xmm7,XMMWORD[80+rdi]
				680	movdqu xmm8,XMMWORD[96+rdi]
				681	movdqu xmm9,XMMWORD[112+rdi]
				682	lea rdi,[128+rdi]
				683	sub rdx,0x80
				684	jmp NEAR $L$ecb_dec_loop8_enter
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	685	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	686	$L$ecb_dec_loop8:
				687	movups XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	688	mov rcx,r11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	689	movdqu xmm2,XMMWORD[rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	690	mov eax,r10d
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	691	movups XMMWORD[16+rsi],xmm3
				692	movdqu xmm3,XMMWORD[16+rdi]
				693	movups XMMWORD[32+rsi],xmm4
				694	movdqu xmm4,XMMWORD[32+rdi]
				695	movups XMMWORD[48+rsi],xmm5
				696	movdqu xmm5,XMMWORD[48+rdi]
				697	movups XMMWORD[64+rsi],xmm6
				698	movdqu xmm6,XMMWORD[64+rdi]
				699	movups XMMWORD[80+rsi],xmm7
				700	movdqu xmm7,XMMWORD[80+rdi]
				701	movups XMMWORD[96+rsi],xmm8
				702	movdqu xmm8,XMMWORD[96+rdi]
				703	movups XMMWORD[112+rsi],xmm9
				704	lea rsi,[128+rsi]
				705	movdqu xmm9,XMMWORD[112+rdi]
				706	lea rdi,[128+rdi]
				707	$L$ecb_dec_loop8_enter:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	708
				709	call _aesni_decrypt8
				710
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	711	movups xmm0,XMMWORD[r11]
				712	sub rdx,0x80
				713	jnc NEAR $L$ecb_dec_loop8
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	714
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	715	movups XMMWORD[rsi],xmm2
				716	pxor xmm2,xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	717	mov rcx,r11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	718	movups XMMWORD[16+rsi],xmm3
				719	pxor xmm3,xmm3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	720	mov eax,r10d
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	721	movups XMMWORD[32+rsi],xmm4
				722	pxor xmm4,xmm4
				723	movups XMMWORD[48+rsi],xmm5
				724	pxor xmm5,xmm5
				725	movups XMMWORD[64+rsi],xmm6
				726	pxor xmm6,xmm6
				727	movups XMMWORD[80+rsi],xmm7
				728	pxor xmm7,xmm7
				729	movups XMMWORD[96+rsi],xmm8
				730	pxor xmm8,xmm8
				731	movups XMMWORD[112+rsi],xmm9
				732	pxor xmm9,xmm9
				733	lea rsi,[128+rsi]
				734	add rdx,0x80
				735	jz NEAR $L$ecb_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	736
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	737	$L$ecb_dec_tail:
				738	movups xmm2,XMMWORD[rdi]
				739	cmp rdx,0x20
				740	jb NEAR $L$ecb_dec_one
				741	movups xmm3,XMMWORD[16+rdi]
				742	je NEAR $L$ecb_dec_two
				743	movups xmm4,XMMWORD[32+rdi]
				744	cmp rdx,0x40
				745	jb NEAR $L$ecb_dec_three
				746	movups xmm5,XMMWORD[48+rdi]
				747	je NEAR $L$ecb_dec_four
				748	movups xmm6,XMMWORD[64+rdi]
				749	cmp rdx,0x60
				750	jb NEAR $L$ecb_dec_five
				751	movups xmm7,XMMWORD[80+rdi]
				752	je NEAR $L$ecb_dec_six
				753	movups xmm8,XMMWORD[96+rdi]
				754	movups xmm0,XMMWORD[rcx]
				755	xorps xmm9,xmm9
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	756	call _aesni_decrypt8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	757	movups XMMWORD[rsi],xmm2
				758	pxor xmm2,xmm2
				759	movups XMMWORD[16+rsi],xmm3
				760	pxor xmm3,xmm3
				761	movups XMMWORD[32+rsi],xmm4
				762	pxor xmm4,xmm4
				763	movups XMMWORD[48+rsi],xmm5
				764	pxor xmm5,xmm5
				765	movups XMMWORD[64+rsi],xmm6
				766	pxor xmm6,xmm6
				767	movups XMMWORD[80+rsi],xmm7
				768	pxor xmm7,xmm7
				769	movups XMMWORD[96+rsi],xmm8
				770	pxor xmm8,xmm8
				771	pxor xmm9,xmm9
				772	jmp NEAR $L$ecb_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	773	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	774	$L$ecb_dec_one:
				775	movups xmm0,XMMWORD[rcx]
				776	movups xmm1,XMMWORD[16+rcx]
				777	lea rcx,[32+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	778	xorps xmm2,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	779	$L$oop_dec1_4:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	780	DB 102,15,56,222,209
				781	dec eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	782	movups xmm1,XMMWORD[rcx]
				783	lea rcx,[16+rcx]
				784	jnz NEAR $L$oop_dec1_4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	785	DB 102,15,56,223,209
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	786	movups XMMWORD[rsi],xmm2
				787	pxor xmm2,xmm2
				788	jmp NEAR $L$ecb_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	789	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	790	$L$ecb_dec_two:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	791	call _aesni_decrypt2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	792	movups XMMWORD[rsi],xmm2
				793	pxor xmm2,xmm2
				794	movups XMMWORD[16+rsi],xmm3
				795	pxor xmm3,xmm3
				796	jmp NEAR $L$ecb_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	797	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	798	$L$ecb_dec_three:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	799	call _aesni_decrypt3
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	800	movups XMMWORD[rsi],xmm2
				801	pxor xmm2,xmm2
				802	movups XMMWORD[16+rsi],xmm3
				803	pxor xmm3,xmm3
				804	movups XMMWORD[32+rsi],xmm4
				805	pxor xmm4,xmm4
				806	jmp NEAR $L$ecb_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	807	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	808	$L$ecb_dec_four:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	809	call _aesni_decrypt4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	810	movups XMMWORD[rsi],xmm2
				811	pxor xmm2,xmm2
				812	movups XMMWORD[16+rsi],xmm3
				813	pxor xmm3,xmm3
				814	movups XMMWORD[32+rsi],xmm4
				815	pxor xmm4,xmm4
				816	movups XMMWORD[48+rsi],xmm5
				817	pxor xmm5,xmm5
				818	jmp NEAR $L$ecb_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	819	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	820	$L$ecb_dec_five:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	821	xorps xmm7,xmm7
				822	call _aesni_decrypt6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	823	movups XMMWORD[rsi],xmm2
				824	pxor xmm2,xmm2
				825	movups XMMWORD[16+rsi],xmm3
				826	pxor xmm3,xmm3
				827	movups XMMWORD[32+rsi],xmm4
				828	pxor xmm4,xmm4
				829	movups XMMWORD[48+rsi],xmm5
				830	pxor xmm5,xmm5
				831	movups XMMWORD[64+rsi],xmm6
				832	pxor xmm6,xmm6
				833	pxor xmm7,xmm7
				834	jmp NEAR $L$ecb_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	835	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	836	$L$ecb_dec_six:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	837	call _aesni_decrypt6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	838	movups XMMWORD[rsi],xmm2
				839	pxor xmm2,xmm2
				840	movups XMMWORD[16+rsi],xmm3
				841	pxor xmm3,xmm3
				842	movups XMMWORD[32+rsi],xmm4
				843	pxor xmm4,xmm4
				844	movups XMMWORD[48+rsi],xmm5
				845	pxor xmm5,xmm5
				846	movups XMMWORD[64+rsi],xmm6
				847	pxor xmm6,xmm6
				848	movups XMMWORD[80+rsi],xmm7
				849	pxor xmm7,xmm7
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	850
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	851	$L$ecb_ret:
				852	xorps xmm0,xmm0
				853	pxor xmm1,xmm1
				854	movaps xmm6,XMMWORD[rsp]
				855	movaps XMMWORD[rsp],xmm0
				856	movaps xmm7,XMMWORD[16+rsp]
				857	movaps XMMWORD[16+rsp],xmm0
				858	movaps xmm8,XMMWORD[32+rsp]
				859	movaps XMMWORD[32+rsp],xmm0
				860	movaps xmm9,XMMWORD[48+rsp]
				861	movaps XMMWORD[48+rsp],xmm0
				862	lea rsp,[88+rsp]
				863	$L$ecb_enc_ret:
				864	mov rdi,QWORD[8+rsp] ;WIN64 epilogue
				865	mov rsi,QWORD[16+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	866	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	867	$L$SEH_end_aesni_ecb_encrypt:
				868	global aesni_ccm64_encrypt_blocks
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	869
				870	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	871	aesni_ccm64_encrypt_blocks:
				872	mov QWORD[8+rsp],rdi ;WIN64 prologue
				873	mov QWORD[16+rsp],rsi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	874	mov rax,rsp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	875	$L$SEH_begin_aesni_ccm64_encrypt_blocks:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	876	mov rdi,rcx
				877	mov rsi,rdx
				878	mov rdx,r8
				879	mov rcx,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	880	mov r8,QWORD[40+rsp]
				881	mov r9,QWORD[48+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	882
				883
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	884	lea rsp,[((-88))+rsp]
				885	movaps XMMWORD[rsp],xmm6
				886	movaps XMMWORD[16+rsp],xmm7
				887	movaps XMMWORD[32+rsp],xmm8
				888	movaps XMMWORD[48+rsp],xmm9
				889	$L$ccm64_enc_body:
				890	mov eax,DWORD[240+rcx]
				891	movdqu xmm6,XMMWORD[r8]
				892	movdqa xmm9,XMMWORD[$L$increment64]
				893	movdqa xmm7,XMMWORD[$L$bswap_mask]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	894
				895	shl eax,4
				896	mov r10d,16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	897	lea r11,[rcx]
				898	movdqu xmm3,XMMWORD[r9]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	899	movdqa xmm2,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	900	lea rcx,[32+rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	901	DB 102,15,56,0,247
				902	sub r10,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	903	jmp NEAR $L$ccm64_enc_outer
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	904	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	905	$L$ccm64_enc_outer:
				906	movups xmm0,XMMWORD[r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	907	mov rax,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	908	movups xmm8,XMMWORD[rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	909
				910	xorps xmm2,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	911	movups xmm1,XMMWORD[16+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	912	xorps xmm0,xmm8
				913	xorps xmm3,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	914	movups xmm0,XMMWORD[32+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	915
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	916	$L$ccm64_enc2_loop:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	917	DB 102,15,56,220,209
				918	DB 102,15,56,220,217
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	919	movups xmm1,XMMWORD[rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	920	add rax,32
				921	DB 102,15,56,220,208
				922	DB 102,15,56,220,216
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	923	movups xmm0,XMMWORD[((-16))+rax*1+rcx]
				924	jnz NEAR $L$ccm64_enc2_loop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	925	DB 102,15,56,220,209
				926	DB 102,15,56,220,217
				927	paddq xmm6,xmm9
				928	dec rdx
				929	DB 102,15,56,221,208
				930	DB 102,15,56,221,216
				931
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	932	lea rdi,[16+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	933	xorps xmm8,xmm2
				934	movdqa xmm2,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	935	movups XMMWORD[rsi],xmm8
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	936	DB 102,15,56,0,215
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	937	lea rsi,[16+rsi]
				938	jnz NEAR $L$ccm64_enc_outer
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	939
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	940	pxor xmm0,xmm0
				941	pxor xmm1,xmm1
				942	pxor xmm2,xmm2
				943	movups XMMWORD[r9],xmm3
				944	pxor xmm3,xmm3
				945	pxor xmm8,xmm8
				946	pxor xmm6,xmm6
				947	movaps xmm6,XMMWORD[rsp]
				948	movaps XMMWORD[rsp],xmm0
				949	movaps xmm7,XMMWORD[16+rsp]
				950	movaps XMMWORD[16+rsp],xmm0
				951	movaps xmm8,XMMWORD[32+rsp]
				952	movaps XMMWORD[32+rsp],xmm0
				953	movaps xmm9,XMMWORD[48+rsp]
				954	movaps XMMWORD[48+rsp],xmm0
				955	lea rsp,[88+rsp]
				956	$L$ccm64_enc_ret:
				957	mov rdi,QWORD[8+rsp] ;WIN64 epilogue
				958	mov rsi,QWORD[16+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	959	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	960	$L$SEH_end_aesni_ccm64_encrypt_blocks:
				961	global aesni_ccm64_decrypt_blocks
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	962
				963	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	964	aesni_ccm64_decrypt_blocks:
				965	mov QWORD[8+rsp],rdi ;WIN64 prologue
				966	mov QWORD[16+rsp],rsi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	967	mov rax,rsp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	968	$L$SEH_begin_aesni_ccm64_decrypt_blocks:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	969	mov rdi,rcx
				970	mov rsi,rdx
				971	mov rdx,r8
				972	mov rcx,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	973	mov r8,QWORD[40+rsp]
				974	mov r9,QWORD[48+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	975
				976
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	977	lea rsp,[((-88))+rsp]
				978	movaps XMMWORD[rsp],xmm6
				979	movaps XMMWORD[16+rsp],xmm7
				980	movaps XMMWORD[32+rsp],xmm8
				981	movaps XMMWORD[48+rsp],xmm9
				982	$L$ccm64_dec_body:
				983	mov eax,DWORD[240+rcx]
				984	movups xmm6,XMMWORD[r8]
				985	movdqu xmm3,XMMWORD[r9]
				986	movdqa xmm9,XMMWORD[$L$increment64]
				987	movdqa xmm7,XMMWORD[$L$bswap_mask]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	988
				989	movaps xmm2,xmm6
				990	mov r10d,eax
				991	mov r11,rcx
				992	DB 102,15,56,0,247
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	993	movups xmm0,XMMWORD[rcx]
				994	movups xmm1,XMMWORD[16+rcx]
				995	lea rcx,[32+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	996	xorps xmm2,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	997	$L$oop_enc1_5:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	998	DB 102,15,56,220,209
				999	dec eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1000	movups xmm1,XMMWORD[rcx]
				1001	lea rcx,[16+rcx]
				1002	jnz NEAR $L$oop_enc1_5
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1003	DB 102,15,56,221,209
				1004	shl r10d,4
				1005	mov eax,16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1006	movups xmm8,XMMWORD[rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1007	paddq xmm6,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1008	lea rdi,[16+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1009	sub rax,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1010	lea rcx,[32+r10*1+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1011	mov r10,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1012	jmp NEAR $L$ccm64_dec_outer
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1013	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1014	$L$ccm64_dec_outer:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1015	xorps xmm8,xmm2
				1016	movdqa xmm2,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1017	movups XMMWORD[rsi],xmm8
				1018	lea rsi,[16+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1019	DB 102,15,56,0,215
				1020
				1021	sub rdx,1
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1022	jz NEAR $L$ccm64_dec_break
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1023
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1024	movups xmm0,XMMWORD[r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1025	mov rax,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1026	movups xmm1,XMMWORD[16+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1027	xorps xmm8,xmm0
				1028	xorps xmm2,xmm0
				1029	xorps xmm3,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1030	movups xmm0,XMMWORD[32+r11]
				1031	jmp NEAR $L$ccm64_dec2_loop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1032	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1033	$L$ccm64_dec2_loop:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1034	DB 102,15,56,220,209
				1035	DB 102,15,56,220,217
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1036	movups xmm1,XMMWORD[rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1037	add rax,32
				1038	DB 102,15,56,220,208
				1039	DB 102,15,56,220,216
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1040	movups xmm0,XMMWORD[((-16))+rax*1+rcx]
				1041	jnz NEAR $L$ccm64_dec2_loop
				1042	movups xmm8,XMMWORD[rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1043	paddq xmm6,xmm9
				1044	DB 102,15,56,220,209
				1045	DB 102,15,56,220,217
				1046	DB 102,15,56,221,208
				1047	DB 102,15,56,221,216
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1048	lea rdi,[16+rdi]
				1049	jmp NEAR $L$ccm64_dec_outer
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1050
				1051	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1052	$L$ccm64_dec_break:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1053
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1054	mov eax,DWORD[240+r11]
				1055	movups xmm0,XMMWORD[r11]
				1056	movups xmm1,XMMWORD[16+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1057	xorps xmm8,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1058	lea r11,[32+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1059	xorps xmm3,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1060	$L$oop_enc1_6:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1061	DB 102,15,56,220,217
				1062	dec eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1063	movups xmm1,XMMWORD[r11]
				1064	lea r11,[16+r11]
				1065	jnz NEAR $L$oop_enc1_6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1066	DB 102,15,56,221,217
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1067	pxor xmm0,xmm0
				1068	pxor xmm1,xmm1
				1069	pxor xmm2,xmm2
				1070	movups XMMWORD[r9],xmm3
				1071	pxor xmm3,xmm3
				1072	pxor xmm8,xmm8
				1073	pxor xmm6,xmm6
				1074	movaps xmm6,XMMWORD[rsp]
				1075	movaps XMMWORD[rsp],xmm0
				1076	movaps xmm7,XMMWORD[16+rsp]
				1077	movaps XMMWORD[16+rsp],xmm0
				1078	movaps xmm8,XMMWORD[32+rsp]
				1079	movaps XMMWORD[32+rsp],xmm0
				1080	movaps xmm9,XMMWORD[48+rsp]
				1081	movaps XMMWORD[48+rsp],xmm0
				1082	lea rsp,[88+rsp]
				1083	$L$ccm64_dec_ret:
				1084	mov rdi,QWORD[8+rsp] ;WIN64 epilogue
				1085	mov rsi,QWORD[16+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1086	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1087	$L$SEH_end_aesni_ccm64_decrypt_blocks:
				1088	global aesni_ctr32_encrypt_blocks
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1089
				1090	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1091	aesni_ctr32_encrypt_blocks:
				1092	mov QWORD[8+rsp],rdi ;WIN64 prologue
				1093	mov QWORD[16+rsp],rsi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1094	mov rax,rsp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1095	$L$SEH_begin_aesni_ctr32_encrypt_blocks:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1096	mov rdi,rcx
				1097	mov rsi,rdx
				1098	mov rdx,r8
				1099	mov rcx,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1100	mov r8,QWORD[40+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1101
				1102
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1103	cmp rdx,1
				1104	jne NEAR $L$ctr32_bulk
				1105
				1106
				1107
				1108	movups xmm2,XMMWORD[r8]
				1109	movups xmm3,XMMWORD[rdi]
				1110	mov edx,DWORD[240+rcx]
				1111	movups xmm0,XMMWORD[rcx]
				1112	movups xmm1,XMMWORD[16+rcx]
				1113	lea rcx,[32+rcx]
				1114	xorps xmm2,xmm0
				1115	$L$oop_enc1_7:
				1116	DB 102,15,56,220,209
				1117	dec edx
				1118	movups xmm1,XMMWORD[rcx]
				1119	lea rcx,[16+rcx]
				1120	jnz NEAR $L$oop_enc1_7
				1121	DB 102,15,56,221,209
				1122	pxor xmm0,xmm0
				1123	pxor xmm1,xmm1
				1124	xorps xmm2,xmm3
				1125	pxor xmm3,xmm3
				1126	movups XMMWORD[rsi],xmm2
				1127	xorps xmm2,xmm2
				1128	jmp NEAR $L$ctr32_epilogue
				1129
				1130	ALIGN 16
				1131	$L$ctr32_bulk:
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1132	lea r11,[rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1133	push rbp
				1134	sub rsp,288
				1135	and rsp,-16
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1136	movaps XMMWORD[(-168)+r11],xmm6
				1137	movaps XMMWORD[(-152)+r11],xmm7
				1138	movaps XMMWORD[(-136)+r11],xmm8
				1139	movaps XMMWORD[(-120)+r11],xmm9
				1140	movaps XMMWORD[(-104)+r11],xmm10
				1141	movaps XMMWORD[(-88)+r11],xmm11
				1142	movaps XMMWORD[(-72)+r11],xmm12
				1143	movaps XMMWORD[(-56)+r11],xmm13
				1144	movaps XMMWORD[(-40)+r11],xmm14
				1145	movaps XMMWORD[(-24)+r11],xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1146	$L$ctr32_body:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1147
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1148
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1149
				1150
				1151	movdqu xmm2,XMMWORD[r8]
				1152	movdqu xmm0,XMMWORD[rcx]
				1153	mov r8d,DWORD[12+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1154	pxor xmm2,xmm0
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1155	mov ebp,DWORD[12+rcx]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1156	movdqa XMMWORD[rsp],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1157	bswap r8d
				1158	movdqa xmm3,xmm2
				1159	movdqa xmm4,xmm2
				1160	movdqa xmm5,xmm2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1161	movdqa XMMWORD[64+rsp],xmm2
				1162	movdqa XMMWORD[80+rsp],xmm2
				1163	movdqa XMMWORD[96+rsp],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1164	mov r10,rdx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1165	movdqa XMMWORD[112+rsp],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1166
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1167	lea rax,[1+r8]
				1168	lea rdx,[2+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1169	bswap eax
				1170	bswap edx
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1171	xor eax,ebp
				1172	xor edx,ebp
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1173	DB 102,15,58,34,216,3
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1174	lea rax,[3+r8]
				1175	movdqa XMMWORD[16+rsp],xmm3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1176	DB 102,15,58,34,226,3
				1177	bswap eax
				1178	mov rdx,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1179	lea r10,[4+r8]
				1180	movdqa XMMWORD[32+rsp],xmm4
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1181	xor eax,ebp
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1182	bswap r10d
				1183	DB 102,15,58,34,232,3
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1184	xor r10d,ebp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1185	movdqa XMMWORD[48+rsp],xmm5
				1186	lea r9,[5+r8]
				1187	mov DWORD[((64+12))+rsp],r10d
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1188	bswap r9d
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1189	lea r10,[6+r8]
				1190	mov eax,DWORD[240+rcx]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1191	xor r9d,ebp
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1192	bswap r10d
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1193	mov DWORD[((80+12))+rsp],r9d
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1194	xor r10d,ebp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1195	lea r9,[7+r8]
				1196	mov DWORD[((96+12))+rsp],r10d
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1197	bswap r9d
Robert Sloan	2424d84	2017-05-01 07:46:28 -0700	[diff] [blame]	1198	lea r10,[OPENSSL_ia32cap_P]
Robert Sloan	572a4e2	2017-04-17 10:52:19 -0700	[diff] [blame]	1199	mov r10d,DWORD[4+r10]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1200	xor r9d,ebp
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1201	and r10d,71303168
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1202	mov DWORD[((112+12))+rsp],r9d
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1203
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1204	movups xmm1,XMMWORD[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1205
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1206	movdqa xmm6,XMMWORD[64+rsp]
				1207	movdqa xmm7,XMMWORD[80+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1208
				1209	cmp rdx,8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1210	jb NEAR $L$ctr32_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1211
				1212	sub rdx,6
				1213	cmp r10d,4194304
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1214	je NEAR $L$ctr32_6x
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1215
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1216	lea rcx,[128+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1217	sub rdx,2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1218	jmp NEAR $L$ctr32_loop8
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1219
				1220	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1221	$L$ctr32_6x:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1222	shl eax,4
				1223	mov r10d,48
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1224	bswap ebp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1225	lea rcx,[32+rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1226	sub r10,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1227	jmp NEAR $L$ctr32_loop6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1228
				1229	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1230	$L$ctr32_loop6:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1231	add r8d,6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1232	movups xmm0,XMMWORD[((-48))+r10*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1233	DB 102,15,56,220,209
				1234	mov eax,r8d
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1235	xor eax,ebp
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1236	DB 102,15,56,220,217
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1237	DB 0x0f,0x38,0xf1,0x44,0x24,12
				1238	lea eax,[1+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1239	DB 102,15,56,220,225
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1240	xor eax,ebp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1241	DB 0x0f,0x38,0xf1,0x44,0x24,28
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1242	DB 102,15,56,220,233
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1243	lea eax,[2+r8]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1244	xor eax,ebp
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1245	DB 102,15,56,220,241
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1246	DB 0x0f,0x38,0xf1,0x44,0x24,44
				1247	lea eax,[3+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1248	DB 102,15,56,220,249
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1249	movups xmm1,XMMWORD[((-32))+r10*1+rcx]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1250	xor eax,ebp
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1251
				1252	DB 102,15,56,220,208
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1253	DB 0x0f,0x38,0xf1,0x44,0x24,60
				1254	lea eax,[4+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1255	DB 102,15,56,220,216
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1256	xor eax,ebp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1257	DB 0x0f,0x38,0xf1,0x44,0x24,76
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1258	DB 102,15,56,220,224
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1259	lea eax,[5+r8]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1260	xor eax,ebp
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1261	DB 102,15,56,220,232
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1262	DB 0x0f,0x38,0xf1,0x44,0x24,92
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1263	mov rax,r10
				1264	DB 102,15,56,220,240
				1265	DB 102,15,56,220,248
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1266	movups xmm0,XMMWORD[((-16))+r10*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1267
				1268	call $L$enc_loop6
				1269
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1270	movdqu xmm8,XMMWORD[rdi]
				1271	movdqu xmm9,XMMWORD[16+rdi]
				1272	movdqu xmm10,XMMWORD[32+rdi]
				1273	movdqu xmm11,XMMWORD[48+rdi]
				1274	movdqu xmm12,XMMWORD[64+rdi]
				1275	movdqu xmm13,XMMWORD[80+rdi]
				1276	lea rdi,[96+rdi]
				1277	movups xmm1,XMMWORD[((-64))+r10*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1278	pxor xmm8,xmm2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1279	movaps xmm2,XMMWORD[rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1280	pxor xmm9,xmm3
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1281	movaps xmm3,XMMWORD[16+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1282	pxor xmm10,xmm4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1283	movaps xmm4,XMMWORD[32+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1284	pxor xmm11,xmm5
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1285	movaps xmm5,XMMWORD[48+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1286	pxor xmm12,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1287	movaps xmm6,XMMWORD[64+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1288	pxor xmm13,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1289	movaps xmm7,XMMWORD[80+rsp]
				1290	movdqu XMMWORD[rsi],xmm8
				1291	movdqu XMMWORD[16+rsi],xmm9
				1292	movdqu XMMWORD[32+rsi],xmm10
				1293	movdqu XMMWORD[48+rsi],xmm11
				1294	movdqu XMMWORD[64+rsi],xmm12
				1295	movdqu XMMWORD[80+rsi],xmm13
				1296	lea rsi,[96+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1297
				1298	sub rdx,6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1299	jnc NEAR $L$ctr32_loop6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1300
				1301	add rdx,6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1302	jz NEAR $L$ctr32_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1303
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1304	lea eax,[((-48))+r10]
				1305	lea rcx,[((-80))+r10*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1306	neg eax
				1307	shr eax,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1308	jmp NEAR $L$ctr32_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1309
				1310	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1311	$L$ctr32_loop8:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1312	add r8d,8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1313	movdqa xmm8,XMMWORD[96+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1314	DB 102,15,56,220,209
				1315	mov r9d,r8d
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1316	movdqa xmm9,XMMWORD[112+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1317	DB 102,15,56,220,217
				1318	bswap r9d
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1319	movups xmm0,XMMWORD[((32-128))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1320	DB 102,15,56,220,225
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1321	xor r9d,ebp
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1322	nop
				1323	DB 102,15,56,220,233
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1324	mov DWORD[((0+12))+rsp],r9d
				1325	lea r9,[1+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1326	DB 102,15,56,220,241
				1327	DB 102,15,56,220,249
				1328	DB 102,68,15,56,220,193
				1329	DB 102,68,15,56,220,201
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1330	movups xmm1,XMMWORD[((48-128))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1331	bswap r9d
				1332	DB 102,15,56,220,208
				1333	DB 102,15,56,220,216
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1334	xor r9d,ebp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1335	DB 0x66,0x90
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1336	DB 102,15,56,220,224
				1337	DB 102,15,56,220,232
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1338	mov DWORD[((16+12))+rsp],r9d
				1339	lea r9,[2+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1340	DB 102,15,56,220,240
				1341	DB 102,15,56,220,248
				1342	DB 102,68,15,56,220,192
				1343	DB 102,68,15,56,220,200
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1344	movups xmm0,XMMWORD[((64-128))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1345	bswap r9d
				1346	DB 102,15,56,220,209
				1347	DB 102,15,56,220,217
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1348	xor r9d,ebp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1349	DB 0x66,0x90
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1350	DB 102,15,56,220,225
				1351	DB 102,15,56,220,233
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1352	mov DWORD[((32+12))+rsp],r9d
				1353	lea r9,[3+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1354	DB 102,15,56,220,241
				1355	DB 102,15,56,220,249
				1356	DB 102,68,15,56,220,193
				1357	DB 102,68,15,56,220,201
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1358	movups xmm1,XMMWORD[((80-128))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1359	bswap r9d
				1360	DB 102,15,56,220,208
				1361	DB 102,15,56,220,216
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1362	xor r9d,ebp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1363	DB 0x66,0x90
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1364	DB 102,15,56,220,224
				1365	DB 102,15,56,220,232
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1366	mov DWORD[((48+12))+rsp],r9d
				1367	lea r9,[4+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1368	DB 102,15,56,220,240
				1369	DB 102,15,56,220,248
				1370	DB 102,68,15,56,220,192
				1371	DB 102,68,15,56,220,200
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1372	movups xmm0,XMMWORD[((96-128))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1373	bswap r9d
				1374	DB 102,15,56,220,209
				1375	DB 102,15,56,220,217
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1376	xor r9d,ebp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1377	DB 0x66,0x90
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1378	DB 102,15,56,220,225
				1379	DB 102,15,56,220,233
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1380	mov DWORD[((64+12))+rsp],r9d
				1381	lea r9,[5+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1382	DB 102,15,56,220,241
				1383	DB 102,15,56,220,249
				1384	DB 102,68,15,56,220,193
				1385	DB 102,68,15,56,220,201
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1386	movups xmm1,XMMWORD[((112-128))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1387	bswap r9d
				1388	DB 102,15,56,220,208
				1389	DB 102,15,56,220,216
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1390	xor r9d,ebp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1391	DB 0x66,0x90
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1392	DB 102,15,56,220,224
				1393	DB 102,15,56,220,232
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1394	mov DWORD[((80+12))+rsp],r9d
				1395	lea r9,[6+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1396	DB 102,15,56,220,240
				1397	DB 102,15,56,220,248
				1398	DB 102,68,15,56,220,192
				1399	DB 102,68,15,56,220,200
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1400	movups xmm0,XMMWORD[((128-128))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1401	bswap r9d
				1402	DB 102,15,56,220,209
				1403	DB 102,15,56,220,217
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1404	xor r9d,ebp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1405	DB 0x66,0x90
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1406	DB 102,15,56,220,225
				1407	DB 102,15,56,220,233
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1408	mov DWORD[((96+12))+rsp],r9d
				1409	lea r9,[7+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1410	DB 102,15,56,220,241
				1411	DB 102,15,56,220,249
				1412	DB 102,68,15,56,220,193
				1413	DB 102,68,15,56,220,201
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1414	movups xmm1,XMMWORD[((144-128))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1415	bswap r9d
				1416	DB 102,15,56,220,208
				1417	DB 102,15,56,220,216
				1418	DB 102,15,56,220,224
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1419	xor r9d,ebp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1420	movdqu xmm10,XMMWORD[rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1421	DB 102,15,56,220,232
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1422	mov DWORD[((112+12))+rsp],r9d
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1423	cmp eax,11
				1424	DB 102,15,56,220,240
				1425	DB 102,15,56,220,248
				1426	DB 102,68,15,56,220,192
				1427	DB 102,68,15,56,220,200
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1428	movups xmm0,XMMWORD[((160-128))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1429
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1430	jb NEAR $L$ctr32_enc_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1431
				1432	DB 102,15,56,220,209
				1433	DB 102,15,56,220,217
				1434	DB 102,15,56,220,225
				1435	DB 102,15,56,220,233
				1436	DB 102,15,56,220,241
				1437	DB 102,15,56,220,249
				1438	DB 102,68,15,56,220,193
				1439	DB 102,68,15,56,220,201
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1440	movups xmm1,XMMWORD[((176-128))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1441
				1442	DB 102,15,56,220,208
				1443	DB 102,15,56,220,216
				1444	DB 102,15,56,220,224
				1445	DB 102,15,56,220,232
				1446	DB 102,15,56,220,240
				1447	DB 102,15,56,220,248
				1448	DB 102,68,15,56,220,192
				1449	DB 102,68,15,56,220,200
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1450	movups xmm0,XMMWORD[((192-128))+rcx]
				1451	je NEAR $L$ctr32_enc_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1452
				1453	DB 102,15,56,220,209
				1454	DB 102,15,56,220,217
				1455	DB 102,15,56,220,225
				1456	DB 102,15,56,220,233
				1457	DB 102,15,56,220,241
				1458	DB 102,15,56,220,249
				1459	DB 102,68,15,56,220,193
				1460	DB 102,68,15,56,220,201
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1461	movups xmm1,XMMWORD[((208-128))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1462
				1463	DB 102,15,56,220,208
				1464	DB 102,15,56,220,216
				1465	DB 102,15,56,220,224
				1466	DB 102,15,56,220,232
				1467	DB 102,15,56,220,240
				1468	DB 102,15,56,220,248
				1469	DB 102,68,15,56,220,192
				1470	DB 102,68,15,56,220,200
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1471	movups xmm0,XMMWORD[((224-128))+rcx]
				1472	jmp NEAR $L$ctr32_enc_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1473
				1474	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1475	$L$ctr32_enc_done:
				1476	movdqu xmm11,XMMWORD[16+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1477	pxor xmm10,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1478	movdqu xmm12,XMMWORD[32+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1479	pxor xmm11,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1480	movdqu xmm13,XMMWORD[48+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1481	pxor xmm12,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1482	movdqu xmm14,XMMWORD[64+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1483	pxor xmm13,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1484	movdqu xmm15,XMMWORD[80+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1485	pxor xmm14,xmm0
				1486	pxor xmm15,xmm0
				1487	DB 102,15,56,220,209
				1488	DB 102,15,56,220,217
				1489	DB 102,15,56,220,225
				1490	DB 102,15,56,220,233
				1491	DB 102,15,56,220,241
				1492	DB 102,15,56,220,249
				1493	DB 102,68,15,56,220,193
				1494	DB 102,68,15,56,220,201
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1495	movdqu xmm1,XMMWORD[96+rdi]
				1496	lea rdi,[128+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1497
				1498	DB 102,65,15,56,221,210
				1499	pxor xmm1,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1500	movdqu xmm10,XMMWORD[((112-128))+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1501	DB 102,65,15,56,221,219
				1502	pxor xmm10,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1503	movdqa xmm11,XMMWORD[rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1504	DB 102,65,15,56,221,228
				1505	DB 102,65,15,56,221,237
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1506	movdqa xmm12,XMMWORD[16+rsp]
				1507	movdqa xmm13,XMMWORD[32+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1508	DB 102,65,15,56,221,246
				1509	DB 102,65,15,56,221,255
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1510	movdqa xmm14,XMMWORD[48+rsp]
				1511	movdqa xmm15,XMMWORD[64+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1512	DB 102,68,15,56,221,193
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1513	movdqa xmm0,XMMWORD[80+rsp]
				1514	movups xmm1,XMMWORD[((16-128))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1515	DB 102,69,15,56,221,202
				1516
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1517	movups XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1518	movdqa xmm2,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1519	movups XMMWORD[16+rsi],xmm3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1520	movdqa xmm3,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1521	movups XMMWORD[32+rsi],xmm4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1522	movdqa xmm4,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1523	movups XMMWORD[48+rsi],xmm5
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1524	movdqa xmm5,xmm14
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1525	movups XMMWORD[64+rsi],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1526	movdqa xmm6,xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1527	movups XMMWORD[80+rsi],xmm7
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1528	movdqa xmm7,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1529	movups XMMWORD[96+rsi],xmm8
				1530	movups XMMWORD[112+rsi],xmm9
				1531	lea rsi,[128+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1532
				1533	sub rdx,8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1534	jnc NEAR $L$ctr32_loop8
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1535
				1536	add rdx,8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1537	jz NEAR $L$ctr32_done
				1538	lea rcx,[((-128))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1539
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1540	$L$ctr32_tail:
				1541
				1542
				1543	lea rcx,[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1544	cmp rdx,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1545	jb NEAR $L$ctr32_loop3
				1546	je NEAR $L$ctr32_loop4
				1547
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1548
				1549	shl eax,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1550	movdqa xmm8,XMMWORD[96+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1551	pxor xmm9,xmm9
				1552
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1553	movups xmm0,XMMWORD[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1554	DB 102,15,56,220,209
				1555	DB 102,15,56,220,217
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1556	lea rcx,[((32-16))+rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1557	neg rax
				1558	DB 102,15,56,220,225
				1559	add rax,16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1560	movups xmm10,XMMWORD[rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1561	DB 102,15,56,220,233
				1562	DB 102,15,56,220,241
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1563	movups xmm11,XMMWORD[16+rdi]
				1564	movups xmm12,XMMWORD[32+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1565	DB 102,15,56,220,249
				1566	DB 102,68,15,56,220,193
				1567
				1568	call $L$enc_loop8_enter
				1569
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1570	movdqu xmm13,XMMWORD[48+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1571	pxor xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1572	movdqu xmm10,XMMWORD[64+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1573	pxor xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1574	movdqu XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1575	pxor xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1576	movdqu XMMWORD[16+rsi],xmm3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1577	pxor xmm5,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1578	movdqu XMMWORD[32+rsi],xmm4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1579	pxor xmm6,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1580	movdqu XMMWORD[48+rsi],xmm5
				1581	movdqu XMMWORD[64+rsi],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1582	cmp rdx,6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1583	jb NEAR $L$ctr32_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1584
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1585	movups xmm11,XMMWORD[80+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1586	xorps xmm7,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1587	movups XMMWORD[80+rsi],xmm7
				1588	je NEAR $L$ctr32_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1589
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1590	movups xmm12,XMMWORD[96+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1591	xorps xmm8,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1592	movups XMMWORD[96+rsi],xmm8
				1593	jmp NEAR $L$ctr32_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1594
				1595	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1596	$L$ctr32_loop4:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1597	DB 102,15,56,220,209
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1598	lea rcx,[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1599	dec eax
				1600	DB 102,15,56,220,217
				1601	DB 102,15,56,220,225
				1602	DB 102,15,56,220,233
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1603	movups xmm1,XMMWORD[rcx]
				1604	jnz NEAR $L$ctr32_loop4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1605	DB 102,15,56,221,209
				1606	DB 102,15,56,221,217
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1607	movups xmm10,XMMWORD[rdi]
				1608	movups xmm11,XMMWORD[16+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1609	DB 102,15,56,221,225
				1610	DB 102,15,56,221,233
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1611	movups xmm12,XMMWORD[32+rdi]
				1612	movups xmm13,XMMWORD[48+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1613
				1614	xorps xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1615	movups XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1616	xorps xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1617	movups XMMWORD[16+rsi],xmm3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1618	pxor xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1619	movdqu XMMWORD[32+rsi],xmm4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1620	pxor xmm5,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1621	movdqu XMMWORD[48+rsi],xmm5
				1622	jmp NEAR $L$ctr32_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1623
				1624	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1625	$L$ctr32_loop3:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1626	DB 102,15,56,220,209
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1627	lea rcx,[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1628	dec eax
				1629	DB 102,15,56,220,217
				1630	DB 102,15,56,220,225
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1631	movups xmm1,XMMWORD[rcx]
				1632	jnz NEAR $L$ctr32_loop3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1633	DB 102,15,56,221,209
				1634	DB 102,15,56,221,217
				1635	DB 102,15,56,221,225
				1636
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1637	movups xmm10,XMMWORD[rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1638	xorps xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1639	movups XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1640	cmp rdx,2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1641	jb NEAR $L$ctr32_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1642
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1643	movups xmm11,XMMWORD[16+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1644	xorps xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1645	movups XMMWORD[16+rsi],xmm3
				1646	je NEAR $L$ctr32_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1647
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1648	movups xmm12,XMMWORD[32+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1649	xorps xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1650	movups XMMWORD[32+rsi],xmm4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1651
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1652	$L$ctr32_done:
				1653	xorps xmm0,xmm0
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1654	xor ebp,ebp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1655	pxor xmm1,xmm1
				1656	pxor xmm2,xmm2
				1657	pxor xmm3,xmm3
				1658	pxor xmm4,xmm4
				1659	pxor xmm5,xmm5
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1660	movaps xmm6,XMMWORD[((-168))+r11]
				1661	movaps XMMWORD[(-168)+r11],xmm0
				1662	movaps xmm7,XMMWORD[((-152))+r11]
				1663	movaps XMMWORD[(-152)+r11],xmm0
				1664	movaps xmm8,XMMWORD[((-136))+r11]
				1665	movaps XMMWORD[(-136)+r11],xmm0
				1666	movaps xmm9,XMMWORD[((-120))+r11]
				1667	movaps XMMWORD[(-120)+r11],xmm0
				1668	movaps xmm10,XMMWORD[((-104))+r11]
				1669	movaps XMMWORD[(-104)+r11],xmm0
				1670	movaps xmm11,XMMWORD[((-88))+r11]
				1671	movaps XMMWORD[(-88)+r11],xmm0
				1672	movaps xmm12,XMMWORD[((-72))+r11]
				1673	movaps XMMWORD[(-72)+r11],xmm0
				1674	movaps xmm13,XMMWORD[((-56))+r11]
				1675	movaps XMMWORD[(-56)+r11],xmm0
				1676	movaps xmm14,XMMWORD[((-40))+r11]
				1677	movaps XMMWORD[(-40)+r11],xmm0
				1678	movaps xmm15,XMMWORD[((-24))+r11]
				1679	movaps XMMWORD[(-24)+r11],xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1680	movaps XMMWORD[rsp],xmm0
				1681	movaps XMMWORD[16+rsp],xmm0
				1682	movaps XMMWORD[32+rsp],xmm0
				1683	movaps XMMWORD[48+rsp],xmm0
				1684	movaps XMMWORD[64+rsp],xmm0
				1685	movaps XMMWORD[80+rsp],xmm0
				1686	movaps XMMWORD[96+rsp],xmm0
				1687	movaps XMMWORD[112+rsp],xmm0
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1688	mov rbp,QWORD[((-8))+r11]
				1689	lea rsp,[r11]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1690	$L$ctr32_epilogue:
				1691	mov rdi,QWORD[8+rsp] ;WIN64 epilogue
				1692	mov rsi,QWORD[16+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1693	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1694	$L$SEH_end_aesni_ctr32_encrypt_blocks:
				1695	global aesni_xts_encrypt
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1696
				1697	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1698	aesni_xts_encrypt:
				1699	mov QWORD[8+rsp],rdi ;WIN64 prologue
				1700	mov QWORD[16+rsp],rsi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1701	mov rax,rsp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1702	$L$SEH_begin_aesni_xts_encrypt:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1703	mov rdi,rcx
				1704	mov rsi,rdx
				1705	mov rdx,r8
				1706	mov rcx,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1707	mov r8,QWORD[40+rsp]
				1708	mov r9,QWORD[48+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1709
				1710
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1711	lea r11,[rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1712	push rbp
				1713	sub rsp,272
				1714	and rsp,-16
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1715	movaps XMMWORD[(-168)+r11],xmm6
				1716	movaps XMMWORD[(-152)+r11],xmm7
				1717	movaps XMMWORD[(-136)+r11],xmm8
				1718	movaps XMMWORD[(-120)+r11],xmm9
				1719	movaps XMMWORD[(-104)+r11],xmm10
				1720	movaps XMMWORD[(-88)+r11],xmm11
				1721	movaps XMMWORD[(-72)+r11],xmm12
				1722	movaps XMMWORD[(-56)+r11],xmm13
				1723	movaps XMMWORD[(-40)+r11],xmm14
				1724	movaps XMMWORD[(-24)+r11],xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1725	$L$xts_enc_body:
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1726	movups xmm2,XMMWORD[r9]
				1727	mov eax,DWORD[240+r8]
				1728	mov r10d,DWORD[240+rcx]
				1729	movups xmm0,XMMWORD[r8]
				1730	movups xmm1,XMMWORD[16+r8]
				1731	lea r8,[32+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1732	xorps xmm2,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1733	$L$oop_enc1_8:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1734	DB 102,15,56,220,209
				1735	dec eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1736	movups xmm1,XMMWORD[r8]
				1737	lea r8,[16+r8]
				1738	jnz NEAR $L$oop_enc1_8
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1739	DB 102,15,56,221,209
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1740	movups xmm0,XMMWORD[rcx]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1741	mov rbp,rcx
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1742	mov eax,r10d
				1743	shl r10d,4
				1744	mov r9,rdx
				1745	and rdx,-16
				1746
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1747	movups xmm1,XMMWORD[16+r10*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1748
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1749	movdqa xmm8,XMMWORD[$L$xts_magic]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1750	movdqa xmm15,xmm2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1751	pshufd xmm9,xmm2,0x5f
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1752	pxor xmm1,xmm0
				1753	movdqa xmm14,xmm9
				1754	paddd xmm9,xmm9
				1755	movdqa xmm10,xmm15
				1756	psrad xmm14,31
				1757	paddq xmm15,xmm15
				1758	pand xmm14,xmm8
				1759	pxor xmm10,xmm0
				1760	pxor xmm15,xmm14
				1761	movdqa xmm14,xmm9
				1762	paddd xmm9,xmm9
				1763	movdqa xmm11,xmm15
				1764	psrad xmm14,31
				1765	paddq xmm15,xmm15
				1766	pand xmm14,xmm8
				1767	pxor xmm11,xmm0
				1768	pxor xmm15,xmm14
				1769	movdqa xmm14,xmm9
				1770	paddd xmm9,xmm9
				1771	movdqa xmm12,xmm15
				1772	psrad xmm14,31
				1773	paddq xmm15,xmm15
				1774	pand xmm14,xmm8
				1775	pxor xmm12,xmm0
				1776	pxor xmm15,xmm14
				1777	movdqa xmm14,xmm9
				1778	paddd xmm9,xmm9
				1779	movdqa xmm13,xmm15
				1780	psrad xmm14,31
				1781	paddq xmm15,xmm15
				1782	pand xmm14,xmm8
				1783	pxor xmm13,xmm0
				1784	pxor xmm15,xmm14
				1785	movdqa xmm14,xmm15
				1786	psrad xmm9,31
				1787	paddq xmm15,xmm15
				1788	pand xmm9,xmm8
				1789	pxor xmm14,xmm0
				1790	pxor xmm15,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1791	movaps XMMWORD[96+rsp],xmm1
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1792
				1793	sub rdx,16*6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1794	jc NEAR $L$xts_enc_short
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1795
				1796	mov eax,16+96
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1797	lea rcx,[32+r10*1+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1798	sub rax,r10
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1799	movups xmm1,XMMWORD[16+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1800	mov r10,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1801	lea r8,[$L$xts_magic]
				1802	jmp NEAR $L$xts_enc_grandloop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1803
				1804	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1805	$L$xts_enc_grandloop:
				1806	movdqu xmm2,XMMWORD[rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1807	movdqa xmm8,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1808	movdqu xmm3,XMMWORD[16+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1809	pxor xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1810	movdqu xmm4,XMMWORD[32+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1811	pxor xmm3,xmm11
				1812	DB 102,15,56,220,209
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1813	movdqu xmm5,XMMWORD[48+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1814	pxor xmm4,xmm12
				1815	DB 102,15,56,220,217
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1816	movdqu xmm6,XMMWORD[64+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1817	pxor xmm5,xmm13
				1818	DB 102,15,56,220,225
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1819	movdqu xmm7,XMMWORD[80+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1820	pxor xmm8,xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1821	movdqa xmm9,XMMWORD[96+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1822	pxor xmm6,xmm14
				1823	DB 102,15,56,220,233
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1824	movups xmm0,XMMWORD[32+rbp]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1825	lea rdi,[96+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1826	pxor xmm7,xmm8
				1827
				1828	pxor xmm10,xmm9
				1829	DB 102,15,56,220,241
				1830	pxor xmm11,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1831	movdqa XMMWORD[rsp],xmm10
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1832	DB 102,15,56,220,249
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1833	movups xmm1,XMMWORD[48+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1834	pxor xmm12,xmm9
				1835
				1836	DB 102,15,56,220,208
				1837	pxor xmm13,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1838	movdqa XMMWORD[16+rsp],xmm11
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1839	DB 102,15,56,220,216
				1840	pxor xmm14,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1841	movdqa XMMWORD[32+rsp],xmm12
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1842	DB 102,15,56,220,224
				1843	DB 102,15,56,220,232
				1844	pxor xmm8,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1845	movdqa XMMWORD[64+rsp],xmm14
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1846	DB 102,15,56,220,240
				1847	DB 102,15,56,220,248
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1848	movups xmm0,XMMWORD[64+rbp]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1849	movdqa XMMWORD[80+rsp],xmm8
				1850	pshufd xmm9,xmm15,0x5f
				1851	jmp NEAR $L$xts_enc_loop6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1852	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1853	$L$xts_enc_loop6:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1854	DB 102,15,56,220,209
				1855	DB 102,15,56,220,217
				1856	DB 102,15,56,220,225
				1857	DB 102,15,56,220,233
				1858	DB 102,15,56,220,241
				1859	DB 102,15,56,220,249
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1860	movups xmm1,XMMWORD[((-64))+rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1861	add rax,32
				1862
				1863	DB 102,15,56,220,208
				1864	DB 102,15,56,220,216
				1865	DB 102,15,56,220,224
				1866	DB 102,15,56,220,232
				1867	DB 102,15,56,220,240
				1868	DB 102,15,56,220,248
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1869	movups xmm0,XMMWORD[((-80))+rax*1+rcx]
				1870	jnz NEAR $L$xts_enc_loop6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1871
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1872	movdqa xmm8,XMMWORD[r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1873	movdqa xmm14,xmm9
				1874	paddd xmm9,xmm9
				1875	DB 102,15,56,220,209
				1876	paddq xmm15,xmm15
				1877	psrad xmm14,31
				1878	DB 102,15,56,220,217
				1879	pand xmm14,xmm8
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1880	movups xmm10,XMMWORD[rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1881	DB 102,15,56,220,225
				1882	DB 102,15,56,220,233
				1883	DB 102,15,56,220,241
				1884	pxor xmm15,xmm14
				1885	movaps xmm11,xmm10
				1886	DB 102,15,56,220,249
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1887	movups xmm1,XMMWORD[((-64))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1888
				1889	movdqa xmm14,xmm9
				1890	DB 102,15,56,220,208
				1891	paddd xmm9,xmm9
				1892	pxor xmm10,xmm15
				1893	DB 102,15,56,220,216
				1894	psrad xmm14,31
				1895	paddq xmm15,xmm15
				1896	DB 102,15,56,220,224
				1897	DB 102,15,56,220,232
				1898	pand xmm14,xmm8
				1899	movaps xmm12,xmm11
				1900	DB 102,15,56,220,240
				1901	pxor xmm15,xmm14
				1902	movdqa xmm14,xmm9
				1903	DB 102,15,56,220,248
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1904	movups xmm0,XMMWORD[((-48))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1905
				1906	paddd xmm9,xmm9
				1907	DB 102,15,56,220,209
				1908	pxor xmm11,xmm15
				1909	psrad xmm14,31
				1910	DB 102,15,56,220,217
				1911	paddq xmm15,xmm15
				1912	pand xmm14,xmm8
				1913	DB 102,15,56,220,225
				1914	DB 102,15,56,220,233
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1915	movdqa XMMWORD[48+rsp],xmm13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1916	pxor xmm15,xmm14
				1917	DB 102,15,56,220,241
				1918	movaps xmm13,xmm12
				1919	movdqa xmm14,xmm9
				1920	DB 102,15,56,220,249
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1921	movups xmm1,XMMWORD[((-32))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1922
				1923	paddd xmm9,xmm9
				1924	DB 102,15,56,220,208
				1925	pxor xmm12,xmm15
				1926	psrad xmm14,31
				1927	DB 102,15,56,220,216
				1928	paddq xmm15,xmm15
				1929	pand xmm14,xmm8
				1930	DB 102,15,56,220,224
				1931	DB 102,15,56,220,232
				1932	DB 102,15,56,220,240
				1933	pxor xmm15,xmm14
				1934	movaps xmm14,xmm13
				1935	DB 102,15,56,220,248
				1936
				1937	movdqa xmm0,xmm9
				1938	paddd xmm9,xmm9
				1939	DB 102,15,56,220,209
				1940	pxor xmm13,xmm15
				1941	psrad xmm0,31
				1942	DB 102,15,56,220,217
				1943	paddq xmm15,xmm15
				1944	pand xmm0,xmm8
				1945	DB 102,15,56,220,225
				1946	DB 102,15,56,220,233
				1947	pxor xmm15,xmm0
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1948	movups xmm0,XMMWORD[rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1949	DB 102,15,56,220,241
				1950	DB 102,15,56,220,249
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1951	movups xmm1,XMMWORD[16+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1952
				1953	pxor xmm14,xmm15
				1954	DB 102,15,56,221,84,36,0
				1955	psrad xmm9,31
				1956	paddq xmm15,xmm15
				1957	DB 102,15,56,221,92,36,16
				1958	DB 102,15,56,221,100,36,32
				1959	pand xmm9,xmm8
				1960	mov rax,r10
				1961	DB 102,15,56,221,108,36,48
				1962	DB 102,15,56,221,116,36,64
				1963	DB 102,15,56,221,124,36,80
				1964	pxor xmm15,xmm9
				1965
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1966	lea rsi,[96+rsi]
				1967	movups XMMWORD[(-96)+rsi],xmm2
				1968	movups XMMWORD[(-80)+rsi],xmm3
				1969	movups XMMWORD[(-64)+rsi],xmm4
				1970	movups XMMWORD[(-48)+rsi],xmm5
				1971	movups XMMWORD[(-32)+rsi],xmm6
				1972	movups XMMWORD[(-16)+rsi],xmm7
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1973	sub rdx,16*6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1974	jnc NEAR $L$xts_enc_grandloop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1975
				1976	mov eax,16+96
				1977	sub eax,r10d
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1978	mov rcx,rbp
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1979	shr eax,4
				1980
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1981	$L$xts_enc_short:
				1982
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1983	mov r10d,eax
				1984	pxor xmm10,xmm0
				1985	add rdx,16*6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1986	jz NEAR $L$xts_enc_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1987
				1988	pxor xmm11,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1989	cmp rdx,0x20
				1990	jb NEAR $L$xts_enc_one
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1991	pxor xmm12,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1992	je NEAR $L$xts_enc_two
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1993
				1994	pxor xmm13,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1995	cmp rdx,0x40
				1996	jb NEAR $L$xts_enc_three
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1997	pxor xmm14,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1998	je NEAR $L$xts_enc_four
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1999
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2000	movdqu xmm2,XMMWORD[rdi]
				2001	movdqu xmm3,XMMWORD[16+rdi]
				2002	movdqu xmm4,XMMWORD[32+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2003	pxor xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2004	movdqu xmm5,XMMWORD[48+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2005	pxor xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2006	movdqu xmm6,XMMWORD[64+rdi]
				2007	lea rdi,[80+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2008	pxor xmm4,xmm12
				2009	pxor xmm5,xmm13
				2010	pxor xmm6,xmm14
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2011	pxor xmm7,xmm7
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2012
				2013	call _aesni_encrypt6
				2014
				2015	xorps xmm2,xmm10
				2016	movdqa xmm10,xmm15
				2017	xorps xmm3,xmm11
				2018	xorps xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2019	movdqu XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2020	xorps xmm5,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2021	movdqu XMMWORD[16+rsi],xmm3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2022	xorps xmm6,xmm14
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2023	movdqu XMMWORD[32+rsi],xmm4
				2024	movdqu XMMWORD[48+rsi],xmm5
				2025	movdqu XMMWORD[64+rsi],xmm6
				2026	lea rsi,[80+rsi]
				2027	jmp NEAR $L$xts_enc_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2028
				2029	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2030	$L$xts_enc_one:
				2031	movups xmm2,XMMWORD[rdi]
				2032	lea rdi,[16+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2033	xorps xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2034	movups xmm0,XMMWORD[rcx]
				2035	movups xmm1,XMMWORD[16+rcx]
				2036	lea rcx,[32+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2037	xorps xmm2,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2038	$L$oop_enc1_9:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2039	DB 102,15,56,220,209
				2040	dec eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2041	movups xmm1,XMMWORD[rcx]
				2042	lea rcx,[16+rcx]
				2043	jnz NEAR $L$oop_enc1_9
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2044	DB 102,15,56,221,209
				2045	xorps xmm2,xmm10
				2046	movdqa xmm10,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2047	movups XMMWORD[rsi],xmm2
				2048	lea rsi,[16+rsi]
				2049	jmp NEAR $L$xts_enc_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2050
				2051	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2052	$L$xts_enc_two:
				2053	movups xmm2,XMMWORD[rdi]
				2054	movups xmm3,XMMWORD[16+rdi]
				2055	lea rdi,[32+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2056	xorps xmm2,xmm10
				2057	xorps xmm3,xmm11
				2058
				2059	call _aesni_encrypt2
				2060
				2061	xorps xmm2,xmm10
				2062	movdqa xmm10,xmm12
				2063	xorps xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2064	movups XMMWORD[rsi],xmm2
				2065	movups XMMWORD[16+rsi],xmm3
				2066	lea rsi,[32+rsi]
				2067	jmp NEAR $L$xts_enc_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2068
				2069	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2070	$L$xts_enc_three:
				2071	movups xmm2,XMMWORD[rdi]
				2072	movups xmm3,XMMWORD[16+rdi]
				2073	movups xmm4,XMMWORD[32+rdi]
				2074	lea rdi,[48+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2075	xorps xmm2,xmm10
				2076	xorps xmm3,xmm11
				2077	xorps xmm4,xmm12
				2078
				2079	call _aesni_encrypt3
				2080
				2081	xorps xmm2,xmm10
				2082	movdqa xmm10,xmm13
				2083	xorps xmm3,xmm11
				2084	xorps xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2085	movups XMMWORD[rsi],xmm2
				2086	movups XMMWORD[16+rsi],xmm3
				2087	movups XMMWORD[32+rsi],xmm4
				2088	lea rsi,[48+rsi]
				2089	jmp NEAR $L$xts_enc_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2090
				2091	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2092	$L$xts_enc_four:
				2093	movups xmm2,XMMWORD[rdi]
				2094	movups xmm3,XMMWORD[16+rdi]
				2095	movups xmm4,XMMWORD[32+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2096	xorps xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2097	movups xmm5,XMMWORD[48+rdi]
				2098	lea rdi,[64+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2099	xorps xmm3,xmm11
				2100	xorps xmm4,xmm12
				2101	xorps xmm5,xmm13
				2102
				2103	call _aesni_encrypt4
				2104
				2105	pxor xmm2,xmm10
				2106	movdqa xmm10,xmm14
				2107	pxor xmm3,xmm11
				2108	pxor xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2109	movdqu XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2110	pxor xmm5,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2111	movdqu XMMWORD[16+rsi],xmm3
				2112	movdqu XMMWORD[32+rsi],xmm4
				2113	movdqu XMMWORD[48+rsi],xmm5
				2114	lea rsi,[64+rsi]
				2115	jmp NEAR $L$xts_enc_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2116
				2117	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2118	$L$xts_enc_done:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2119	and r9,15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2120	jz NEAR $L$xts_enc_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2121	mov rdx,r9
				2122
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2123	$L$xts_enc_steal:
				2124	movzx eax,BYTE[rdi]
				2125	movzx ecx,BYTE[((-16))+rsi]
				2126	lea rdi,[1+rdi]
				2127	mov BYTE[((-16))+rsi],al
				2128	mov BYTE[rsi],cl
				2129	lea rsi,[1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2130	sub rdx,1
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2131	jnz NEAR $L$xts_enc_steal
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2132
				2133	sub rsi,r9
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2134	mov rcx,rbp
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2135	mov eax,r10d
				2136
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2137	movups xmm2,XMMWORD[((-16))+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2138	xorps xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2139	movups xmm0,XMMWORD[rcx]
				2140	movups xmm1,XMMWORD[16+rcx]
				2141	lea rcx,[32+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2142	xorps xmm2,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2143	$L$oop_enc1_10:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2144	DB 102,15,56,220,209
				2145	dec eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2146	movups xmm1,XMMWORD[rcx]
				2147	lea rcx,[16+rcx]
				2148	jnz NEAR $L$oop_enc1_10
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2149	DB 102,15,56,221,209
				2150	xorps xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2151	movups XMMWORD[(-16)+rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2152
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2153	$L$xts_enc_ret:
				2154	xorps xmm0,xmm0
				2155	pxor xmm1,xmm1
				2156	pxor xmm2,xmm2
				2157	pxor xmm3,xmm3
				2158	pxor xmm4,xmm4
				2159	pxor xmm5,xmm5
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2160	movaps xmm6,XMMWORD[((-168))+r11]
				2161	movaps XMMWORD[(-168)+r11],xmm0
				2162	movaps xmm7,XMMWORD[((-152))+r11]
				2163	movaps XMMWORD[(-152)+r11],xmm0
				2164	movaps xmm8,XMMWORD[((-136))+r11]
				2165	movaps XMMWORD[(-136)+r11],xmm0
				2166	movaps xmm9,XMMWORD[((-120))+r11]
				2167	movaps XMMWORD[(-120)+r11],xmm0
				2168	movaps xmm10,XMMWORD[((-104))+r11]
				2169	movaps XMMWORD[(-104)+r11],xmm0
				2170	movaps xmm11,XMMWORD[((-88))+r11]
				2171	movaps XMMWORD[(-88)+r11],xmm0
				2172	movaps xmm12,XMMWORD[((-72))+r11]
				2173	movaps XMMWORD[(-72)+r11],xmm0
				2174	movaps xmm13,XMMWORD[((-56))+r11]
				2175	movaps XMMWORD[(-56)+r11],xmm0
				2176	movaps xmm14,XMMWORD[((-40))+r11]
				2177	movaps XMMWORD[(-40)+r11],xmm0
				2178	movaps xmm15,XMMWORD[((-24))+r11]
				2179	movaps XMMWORD[(-24)+r11],xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2180	movaps XMMWORD[rsp],xmm0
				2181	movaps XMMWORD[16+rsp],xmm0
				2182	movaps XMMWORD[32+rsp],xmm0
				2183	movaps XMMWORD[48+rsp],xmm0
				2184	movaps XMMWORD[64+rsp],xmm0
				2185	movaps XMMWORD[80+rsp],xmm0
				2186	movaps XMMWORD[96+rsp],xmm0
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2187	mov rbp,QWORD[((-8))+r11]
				2188	lea rsp,[r11]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2189	$L$xts_enc_epilogue:
				2190	mov rdi,QWORD[8+rsp] ;WIN64 epilogue
				2191	mov rsi,QWORD[16+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2192	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2193	$L$SEH_end_aesni_xts_encrypt:
				2194	global aesni_xts_decrypt
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2195
				2196	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2197	aesni_xts_decrypt:
				2198	mov QWORD[8+rsp],rdi ;WIN64 prologue
				2199	mov QWORD[16+rsp],rsi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2200	mov rax,rsp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2201	$L$SEH_begin_aesni_xts_decrypt:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2202	mov rdi,rcx
				2203	mov rsi,rdx
				2204	mov rdx,r8
				2205	mov rcx,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2206	mov r8,QWORD[40+rsp]
				2207	mov r9,QWORD[48+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2208
				2209
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2210	lea r11,[rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2211	push rbp
				2212	sub rsp,272
				2213	and rsp,-16
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2214	movaps XMMWORD[(-168)+r11],xmm6
				2215	movaps XMMWORD[(-152)+r11],xmm7
				2216	movaps XMMWORD[(-136)+r11],xmm8
				2217	movaps XMMWORD[(-120)+r11],xmm9
				2218	movaps XMMWORD[(-104)+r11],xmm10
				2219	movaps XMMWORD[(-88)+r11],xmm11
				2220	movaps XMMWORD[(-72)+r11],xmm12
				2221	movaps XMMWORD[(-56)+r11],xmm13
				2222	movaps XMMWORD[(-40)+r11],xmm14
				2223	movaps XMMWORD[(-24)+r11],xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2224	$L$xts_dec_body:
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2225	movups xmm2,XMMWORD[r9]
				2226	mov eax,DWORD[240+r8]
				2227	mov r10d,DWORD[240+rcx]
				2228	movups xmm0,XMMWORD[r8]
				2229	movups xmm1,XMMWORD[16+r8]
				2230	lea r8,[32+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2231	xorps xmm2,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2232	$L$oop_enc1_11:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2233	DB 102,15,56,220,209
				2234	dec eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2235	movups xmm1,XMMWORD[r8]
				2236	lea r8,[16+r8]
				2237	jnz NEAR $L$oop_enc1_11
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2238	DB 102,15,56,221,209
				2239	xor eax,eax
				2240	test rdx,15
				2241	setnz al
				2242	shl rax,4
				2243	sub rdx,rax
				2244
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2245	movups xmm0,XMMWORD[rcx]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2246	mov rbp,rcx
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2247	mov eax,r10d
				2248	shl r10d,4
				2249	mov r9,rdx
				2250	and rdx,-16
				2251
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2252	movups xmm1,XMMWORD[16+r10*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2253
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2254	movdqa xmm8,XMMWORD[$L$xts_magic]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2255	movdqa xmm15,xmm2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2256	pshufd xmm9,xmm2,0x5f
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2257	pxor xmm1,xmm0
				2258	movdqa xmm14,xmm9
				2259	paddd xmm9,xmm9
				2260	movdqa xmm10,xmm15
				2261	psrad xmm14,31
				2262	paddq xmm15,xmm15
				2263	pand xmm14,xmm8
				2264	pxor xmm10,xmm0
				2265	pxor xmm15,xmm14
				2266	movdqa xmm14,xmm9
				2267	paddd xmm9,xmm9
				2268	movdqa xmm11,xmm15
				2269	psrad xmm14,31
				2270	paddq xmm15,xmm15
				2271	pand xmm14,xmm8
				2272	pxor xmm11,xmm0
				2273	pxor xmm15,xmm14
				2274	movdqa xmm14,xmm9
				2275	paddd xmm9,xmm9
				2276	movdqa xmm12,xmm15
				2277	psrad xmm14,31
				2278	paddq xmm15,xmm15
				2279	pand xmm14,xmm8
				2280	pxor xmm12,xmm0
				2281	pxor xmm15,xmm14
				2282	movdqa xmm14,xmm9
				2283	paddd xmm9,xmm9
				2284	movdqa xmm13,xmm15
				2285	psrad xmm14,31
				2286	paddq xmm15,xmm15
				2287	pand xmm14,xmm8
				2288	pxor xmm13,xmm0
				2289	pxor xmm15,xmm14
				2290	movdqa xmm14,xmm15
				2291	psrad xmm9,31
				2292	paddq xmm15,xmm15
				2293	pand xmm9,xmm8
				2294	pxor xmm14,xmm0
				2295	pxor xmm15,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2296	movaps XMMWORD[96+rsp],xmm1
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2297
				2298	sub rdx,16*6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2299	jc NEAR $L$xts_dec_short
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2300
				2301	mov eax,16+96
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2302	lea rcx,[32+r10*1+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2303	sub rax,r10
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2304	movups xmm1,XMMWORD[16+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2305	mov r10,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2306	lea r8,[$L$xts_magic]
				2307	jmp NEAR $L$xts_dec_grandloop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2308
				2309	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2310	$L$xts_dec_grandloop:
				2311	movdqu xmm2,XMMWORD[rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2312	movdqa xmm8,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2313	movdqu xmm3,XMMWORD[16+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2314	pxor xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2315	movdqu xmm4,XMMWORD[32+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2316	pxor xmm3,xmm11
				2317	DB 102,15,56,222,209
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2318	movdqu xmm5,XMMWORD[48+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2319	pxor xmm4,xmm12
				2320	DB 102,15,56,222,217
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2321	movdqu xmm6,XMMWORD[64+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2322	pxor xmm5,xmm13
				2323	DB 102,15,56,222,225
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2324	movdqu xmm7,XMMWORD[80+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2325	pxor xmm8,xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2326	movdqa xmm9,XMMWORD[96+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2327	pxor xmm6,xmm14
				2328	DB 102,15,56,222,233
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2329	movups xmm0,XMMWORD[32+rbp]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2330	lea rdi,[96+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2331	pxor xmm7,xmm8
				2332
				2333	pxor xmm10,xmm9
				2334	DB 102,15,56,222,241
				2335	pxor xmm11,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2336	movdqa XMMWORD[rsp],xmm10
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2337	DB 102,15,56,222,249
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2338	movups xmm1,XMMWORD[48+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2339	pxor xmm12,xmm9
				2340
				2341	DB 102,15,56,222,208
				2342	pxor xmm13,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2343	movdqa XMMWORD[16+rsp],xmm11
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2344	DB 102,15,56,222,216
				2345	pxor xmm14,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2346	movdqa XMMWORD[32+rsp],xmm12
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2347	DB 102,15,56,222,224
				2348	DB 102,15,56,222,232
				2349	pxor xmm8,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2350	movdqa XMMWORD[64+rsp],xmm14
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2351	DB 102,15,56,222,240
				2352	DB 102,15,56,222,248
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2353	movups xmm0,XMMWORD[64+rbp]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2354	movdqa XMMWORD[80+rsp],xmm8
				2355	pshufd xmm9,xmm15,0x5f
				2356	jmp NEAR $L$xts_dec_loop6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2357	ALIGN 32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2358	$L$xts_dec_loop6:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2359	DB 102,15,56,222,209
				2360	DB 102,15,56,222,217
				2361	DB 102,15,56,222,225
				2362	DB 102,15,56,222,233
				2363	DB 102,15,56,222,241
				2364	DB 102,15,56,222,249
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2365	movups xmm1,XMMWORD[((-64))+rax*1+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2366	add rax,32
				2367
				2368	DB 102,15,56,222,208
				2369	DB 102,15,56,222,216
				2370	DB 102,15,56,222,224
				2371	DB 102,15,56,222,232
				2372	DB 102,15,56,222,240
				2373	DB 102,15,56,222,248
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2374	movups xmm0,XMMWORD[((-80))+rax*1+rcx]
				2375	jnz NEAR $L$xts_dec_loop6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2376
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2377	movdqa xmm8,XMMWORD[r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2378	movdqa xmm14,xmm9
				2379	paddd xmm9,xmm9
				2380	DB 102,15,56,222,209
				2381	paddq xmm15,xmm15
				2382	psrad xmm14,31
				2383	DB 102,15,56,222,217
				2384	pand xmm14,xmm8
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2385	movups xmm10,XMMWORD[rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2386	DB 102,15,56,222,225
				2387	DB 102,15,56,222,233
				2388	DB 102,15,56,222,241
				2389	pxor xmm15,xmm14
				2390	movaps xmm11,xmm10
				2391	DB 102,15,56,222,249
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2392	movups xmm1,XMMWORD[((-64))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2393
				2394	movdqa xmm14,xmm9
				2395	DB 102,15,56,222,208
				2396	paddd xmm9,xmm9
				2397	pxor xmm10,xmm15
				2398	DB 102,15,56,222,216
				2399	psrad xmm14,31
				2400	paddq xmm15,xmm15
				2401	DB 102,15,56,222,224
				2402	DB 102,15,56,222,232
				2403	pand xmm14,xmm8
				2404	movaps xmm12,xmm11
				2405	DB 102,15,56,222,240
				2406	pxor xmm15,xmm14
				2407	movdqa xmm14,xmm9
				2408	DB 102,15,56,222,248
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2409	movups xmm0,XMMWORD[((-48))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2410
				2411	paddd xmm9,xmm9
				2412	DB 102,15,56,222,209
				2413	pxor xmm11,xmm15
				2414	psrad xmm14,31
				2415	DB 102,15,56,222,217
				2416	paddq xmm15,xmm15
				2417	pand xmm14,xmm8
				2418	DB 102,15,56,222,225
				2419	DB 102,15,56,222,233
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2420	movdqa XMMWORD[48+rsp],xmm13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2421	pxor xmm15,xmm14
				2422	DB 102,15,56,222,241
				2423	movaps xmm13,xmm12
				2424	movdqa xmm14,xmm9
				2425	DB 102,15,56,222,249
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2426	movups xmm1,XMMWORD[((-32))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2427
				2428	paddd xmm9,xmm9
				2429	DB 102,15,56,222,208
				2430	pxor xmm12,xmm15
				2431	psrad xmm14,31
				2432	DB 102,15,56,222,216
				2433	paddq xmm15,xmm15
				2434	pand xmm14,xmm8
				2435	DB 102,15,56,222,224
				2436	DB 102,15,56,222,232
				2437	DB 102,15,56,222,240
				2438	pxor xmm15,xmm14
				2439	movaps xmm14,xmm13
				2440	DB 102,15,56,222,248
				2441
				2442	movdqa xmm0,xmm9
				2443	paddd xmm9,xmm9
				2444	DB 102,15,56,222,209
				2445	pxor xmm13,xmm15
				2446	psrad xmm0,31
				2447	DB 102,15,56,222,217
				2448	paddq xmm15,xmm15
				2449	pand xmm0,xmm8
				2450	DB 102,15,56,222,225
				2451	DB 102,15,56,222,233
				2452	pxor xmm15,xmm0
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2453	movups xmm0,XMMWORD[rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2454	DB 102,15,56,222,241
				2455	DB 102,15,56,222,249
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2456	movups xmm1,XMMWORD[16+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2457
				2458	pxor xmm14,xmm15
				2459	DB 102,15,56,223,84,36,0
				2460	psrad xmm9,31
				2461	paddq xmm15,xmm15
				2462	DB 102,15,56,223,92,36,16
				2463	DB 102,15,56,223,100,36,32
				2464	pand xmm9,xmm8
				2465	mov rax,r10
				2466	DB 102,15,56,223,108,36,48
				2467	DB 102,15,56,223,116,36,64
				2468	DB 102,15,56,223,124,36,80
				2469	pxor xmm15,xmm9
				2470
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2471	lea rsi,[96+rsi]
				2472	movups XMMWORD[(-96)+rsi],xmm2
				2473	movups XMMWORD[(-80)+rsi],xmm3
				2474	movups XMMWORD[(-64)+rsi],xmm4
				2475	movups XMMWORD[(-48)+rsi],xmm5
				2476	movups XMMWORD[(-32)+rsi],xmm6
				2477	movups XMMWORD[(-16)+rsi],xmm7
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2478	sub rdx,16*6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2479	jnc NEAR $L$xts_dec_grandloop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2480
				2481	mov eax,16+96
				2482	sub eax,r10d
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2483	mov rcx,rbp
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2484	shr eax,4
				2485
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2486	$L$xts_dec_short:
				2487
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2488	mov r10d,eax
				2489	pxor xmm10,xmm0
				2490	pxor xmm11,xmm0
				2491	add rdx,16*6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2492	jz NEAR $L$xts_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2493
				2494	pxor xmm12,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2495	cmp rdx,0x20
				2496	jb NEAR $L$xts_dec_one
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2497	pxor xmm13,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2498	je NEAR $L$xts_dec_two
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2499
				2500	pxor xmm14,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2501	cmp rdx,0x40
				2502	jb NEAR $L$xts_dec_three
				2503	je NEAR $L$xts_dec_four
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2504
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2505	movdqu xmm2,XMMWORD[rdi]
				2506	movdqu xmm3,XMMWORD[16+rdi]
				2507	movdqu xmm4,XMMWORD[32+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2508	pxor xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2509	movdqu xmm5,XMMWORD[48+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2510	pxor xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2511	movdqu xmm6,XMMWORD[64+rdi]
				2512	lea rdi,[80+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2513	pxor xmm4,xmm12
				2514	pxor xmm5,xmm13
				2515	pxor xmm6,xmm14
				2516
				2517	call _aesni_decrypt6
				2518
				2519	xorps xmm2,xmm10
				2520	xorps xmm3,xmm11
				2521	xorps xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2522	movdqu XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2523	xorps xmm5,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2524	movdqu XMMWORD[16+rsi],xmm3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2525	xorps xmm6,xmm14
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2526	movdqu XMMWORD[32+rsi],xmm4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2527	pxor xmm14,xmm14
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2528	movdqu XMMWORD[48+rsi],xmm5
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2529	pcmpgtd xmm14,xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2530	movdqu XMMWORD[64+rsi],xmm6
				2531	lea rsi,[80+rsi]
				2532	pshufd xmm11,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2533	and r9,15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2534	jz NEAR $L$xts_dec_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2535
				2536	movdqa xmm10,xmm15
				2537	paddq xmm15,xmm15
				2538	pand xmm11,xmm8
				2539	pxor xmm11,xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2540	jmp NEAR $L$xts_dec_done2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2541
				2542	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2543	$L$xts_dec_one:
				2544	movups xmm2,XMMWORD[rdi]
				2545	lea rdi,[16+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2546	xorps xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2547	movups xmm0,XMMWORD[rcx]
				2548	movups xmm1,XMMWORD[16+rcx]
				2549	lea rcx,[32+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2550	xorps xmm2,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2551	$L$oop_dec1_12:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2552	DB 102,15,56,222,209
				2553	dec eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2554	movups xmm1,XMMWORD[rcx]
				2555	lea rcx,[16+rcx]
				2556	jnz NEAR $L$oop_dec1_12
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2557	DB 102,15,56,223,209
				2558	xorps xmm2,xmm10
				2559	movdqa xmm10,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2560	movups XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2561	movdqa xmm11,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2562	lea rsi,[16+rsi]
				2563	jmp NEAR $L$xts_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2564
				2565	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2566	$L$xts_dec_two:
				2567	movups xmm2,XMMWORD[rdi]
				2568	movups xmm3,XMMWORD[16+rdi]
				2569	lea rdi,[32+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2570	xorps xmm2,xmm10
				2571	xorps xmm3,xmm11
				2572
				2573	call _aesni_decrypt2
				2574
				2575	xorps xmm2,xmm10
				2576	movdqa xmm10,xmm12
				2577	xorps xmm3,xmm11
				2578	movdqa xmm11,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2579	movups XMMWORD[rsi],xmm2
				2580	movups XMMWORD[16+rsi],xmm3
				2581	lea rsi,[32+rsi]
				2582	jmp NEAR $L$xts_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2583
				2584	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2585	$L$xts_dec_three:
				2586	movups xmm2,XMMWORD[rdi]
				2587	movups xmm3,XMMWORD[16+rdi]
				2588	movups xmm4,XMMWORD[32+rdi]
				2589	lea rdi,[48+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2590	xorps xmm2,xmm10
				2591	xorps xmm3,xmm11
				2592	xorps xmm4,xmm12
				2593
				2594	call _aesni_decrypt3
				2595
				2596	xorps xmm2,xmm10
				2597	movdqa xmm10,xmm13
				2598	xorps xmm3,xmm11
				2599	movdqa xmm11,xmm14
				2600	xorps xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2601	movups XMMWORD[rsi],xmm2
				2602	movups XMMWORD[16+rsi],xmm3
				2603	movups XMMWORD[32+rsi],xmm4
				2604	lea rsi,[48+rsi]
				2605	jmp NEAR $L$xts_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2606
				2607	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2608	$L$xts_dec_four:
				2609	movups xmm2,XMMWORD[rdi]
				2610	movups xmm3,XMMWORD[16+rdi]
				2611	movups xmm4,XMMWORD[32+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2612	xorps xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2613	movups xmm5,XMMWORD[48+rdi]
				2614	lea rdi,[64+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2615	xorps xmm3,xmm11
				2616	xorps xmm4,xmm12
				2617	xorps xmm5,xmm13
				2618
				2619	call _aesni_decrypt4
				2620
				2621	pxor xmm2,xmm10
				2622	movdqa xmm10,xmm14
				2623	pxor xmm3,xmm11
				2624	movdqa xmm11,xmm15
				2625	pxor xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2626	movdqu XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2627	pxor xmm5,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2628	movdqu XMMWORD[16+rsi],xmm3
				2629	movdqu XMMWORD[32+rsi],xmm4
				2630	movdqu XMMWORD[48+rsi],xmm5
				2631	lea rsi,[64+rsi]
				2632	jmp NEAR $L$xts_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2633
				2634	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2635	$L$xts_dec_done:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2636	and r9,15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2637	jz NEAR $L$xts_dec_ret
				2638	$L$xts_dec_done2:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2639	mov rdx,r9
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2640	mov rcx,rbp
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2641	mov eax,r10d
				2642
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2643	movups xmm2,XMMWORD[rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2644	xorps xmm2,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2645	movups xmm0,XMMWORD[rcx]
				2646	movups xmm1,XMMWORD[16+rcx]
				2647	lea rcx,[32+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2648	xorps xmm2,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2649	$L$oop_dec1_13:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2650	DB 102,15,56,222,209
				2651	dec eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2652	movups xmm1,XMMWORD[rcx]
				2653	lea rcx,[16+rcx]
				2654	jnz NEAR $L$oop_dec1_13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2655	DB 102,15,56,223,209
				2656	xorps xmm2,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2657	movups XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2658
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2659	$L$xts_dec_steal:
				2660	movzx eax,BYTE[16+rdi]
				2661	movzx ecx,BYTE[rsi]
				2662	lea rdi,[1+rdi]
				2663	mov BYTE[rsi],al
				2664	mov BYTE[16+rsi],cl
				2665	lea rsi,[1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2666	sub rdx,1
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2667	jnz NEAR $L$xts_dec_steal
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2668
				2669	sub rsi,r9
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2670	mov rcx,rbp
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2671	mov eax,r10d
				2672
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2673	movups xmm2,XMMWORD[rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2674	xorps xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2675	movups xmm0,XMMWORD[rcx]
				2676	movups xmm1,XMMWORD[16+rcx]
				2677	lea rcx,[32+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2678	xorps xmm2,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2679	$L$oop_dec1_14:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2680	DB 102,15,56,222,209
				2681	dec eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2682	movups xmm1,XMMWORD[rcx]
				2683	lea rcx,[16+rcx]
				2684	jnz NEAR $L$oop_dec1_14
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2685	DB 102,15,56,223,209
				2686	xorps xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2687	movups XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2688
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2689	$L$xts_dec_ret:
				2690	xorps xmm0,xmm0
				2691	pxor xmm1,xmm1
				2692	pxor xmm2,xmm2
				2693	pxor xmm3,xmm3
				2694	pxor xmm4,xmm4
				2695	pxor xmm5,xmm5
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2696	movaps xmm6,XMMWORD[((-168))+r11]
				2697	movaps XMMWORD[(-168)+r11],xmm0
				2698	movaps xmm7,XMMWORD[((-152))+r11]
				2699	movaps XMMWORD[(-152)+r11],xmm0
				2700	movaps xmm8,XMMWORD[((-136))+r11]
				2701	movaps XMMWORD[(-136)+r11],xmm0
				2702	movaps xmm9,XMMWORD[((-120))+r11]
				2703	movaps XMMWORD[(-120)+r11],xmm0
				2704	movaps xmm10,XMMWORD[((-104))+r11]
				2705	movaps XMMWORD[(-104)+r11],xmm0
				2706	movaps xmm11,XMMWORD[((-88))+r11]
				2707	movaps XMMWORD[(-88)+r11],xmm0
				2708	movaps xmm12,XMMWORD[((-72))+r11]
				2709	movaps XMMWORD[(-72)+r11],xmm0
				2710	movaps xmm13,XMMWORD[((-56))+r11]
				2711	movaps XMMWORD[(-56)+r11],xmm0
				2712	movaps xmm14,XMMWORD[((-40))+r11]
				2713	movaps XMMWORD[(-40)+r11],xmm0
				2714	movaps xmm15,XMMWORD[((-24))+r11]
				2715	movaps XMMWORD[(-24)+r11],xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2716	movaps XMMWORD[rsp],xmm0
				2717	movaps XMMWORD[16+rsp],xmm0
				2718	movaps XMMWORD[32+rsp],xmm0
				2719	movaps XMMWORD[48+rsp],xmm0
				2720	movaps XMMWORD[64+rsp],xmm0
				2721	movaps XMMWORD[80+rsp],xmm0
				2722	movaps XMMWORD[96+rsp],xmm0
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2723	mov rbp,QWORD[((-8))+r11]
				2724	lea rsp,[r11]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2725	$L$xts_dec_epilogue:
				2726	mov rdi,QWORD[8+rsp] ;WIN64 epilogue
				2727	mov rsi,QWORD[16+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2728	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2729	$L$SEH_end_aesni_xts_decrypt:
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2730	global aesni_ocb_encrypt
				2731
				2732	ALIGN 32
				2733	aesni_ocb_encrypt:
				2734	mov QWORD[8+rsp],rdi ;WIN64 prologue
				2735	mov QWORD[16+rsp],rsi
				2736	mov rax,rsp
				2737	$L$SEH_begin_aesni_ocb_encrypt:
				2738	mov rdi,rcx
				2739	mov rsi,rdx
				2740	mov rdx,r8
				2741	mov rcx,r9
				2742	mov r8,QWORD[40+rsp]
				2743	mov r9,QWORD[48+rsp]
				2744
				2745
				2746	lea rax,[rsp]
				2747	push rbx
				2748	push rbp
				2749	push r12
				2750	push r13
				2751	push r14
				2752	lea rsp,[((-160))+rsp]
				2753	movaps XMMWORD[rsp],xmm6
				2754	movaps XMMWORD[16+rsp],xmm7
				2755	movaps XMMWORD[32+rsp],xmm8
				2756	movaps XMMWORD[48+rsp],xmm9
				2757	movaps XMMWORD[64+rsp],xmm10
				2758	movaps XMMWORD[80+rsp],xmm11
				2759	movaps XMMWORD[96+rsp],xmm12
				2760	movaps XMMWORD[112+rsp],xmm13
				2761	movaps XMMWORD[128+rsp],xmm14
				2762	movaps XMMWORD[144+rsp],xmm15
				2763	$L$ocb_enc_body:
				2764	mov rbx,QWORD[56+rax]
				2765	mov rbp,QWORD[((56+8))+rax]
				2766
				2767	mov r10d,DWORD[240+rcx]
				2768	mov r11,rcx
				2769	shl r10d,4
				2770	movups xmm9,XMMWORD[rcx]
				2771	movups xmm1,XMMWORD[16+r10*1+rcx]
				2772
				2773	movdqu xmm15,XMMWORD[r9]
				2774	pxor xmm9,xmm1
				2775	pxor xmm15,xmm1
				2776
				2777	mov eax,16+32
				2778	lea rcx,[32+r10*1+r11]
				2779	movups xmm1,XMMWORD[16+r11]
				2780	sub rax,r10
				2781	mov r10,rax
				2782
				2783	movdqu xmm10,XMMWORD[rbx]
				2784	movdqu xmm8,XMMWORD[rbp]
				2785
				2786	test r8,1
				2787	jnz NEAR $L$ocb_enc_odd
				2788
				2789	bsf r12,r8
				2790	add r8,1
				2791	shl r12,4
				2792	movdqu xmm7,XMMWORD[r12*1+rbx]
				2793	movdqu xmm2,XMMWORD[rdi]
				2794	lea rdi,[16+rdi]
				2795
				2796	call __ocb_encrypt1
				2797
				2798	movdqa xmm15,xmm7
				2799	movups XMMWORD[rsi],xmm2
				2800	lea rsi,[16+rsi]
				2801	sub rdx,1
				2802	jz NEAR $L$ocb_enc_done
				2803
				2804	$L$ocb_enc_odd:
				2805	lea r12,[1+r8]
				2806	lea r13,[3+r8]
				2807	lea r14,[5+r8]
				2808	lea r8,[6+r8]
				2809	bsf r12,r12
				2810	bsf r13,r13
				2811	bsf r14,r14
				2812	shl r12,4
				2813	shl r13,4
				2814	shl r14,4
				2815
				2816	sub rdx,6
				2817	jc NEAR $L$ocb_enc_short
				2818	jmp NEAR $L$ocb_enc_grandloop
				2819
				2820	ALIGN 32
				2821	$L$ocb_enc_grandloop:
				2822	movdqu xmm2,XMMWORD[rdi]
				2823	movdqu xmm3,XMMWORD[16+rdi]
				2824	movdqu xmm4,XMMWORD[32+rdi]
				2825	movdqu xmm5,XMMWORD[48+rdi]
				2826	movdqu xmm6,XMMWORD[64+rdi]
				2827	movdqu xmm7,XMMWORD[80+rdi]
				2828	lea rdi,[96+rdi]
				2829
				2830	call __ocb_encrypt6
				2831
				2832	movups XMMWORD[rsi],xmm2
				2833	movups XMMWORD[16+rsi],xmm3
				2834	movups XMMWORD[32+rsi],xmm4
				2835	movups XMMWORD[48+rsi],xmm5
				2836	movups XMMWORD[64+rsi],xmm6
				2837	movups XMMWORD[80+rsi],xmm7
				2838	lea rsi,[96+rsi]
				2839	sub rdx,6
				2840	jnc NEAR $L$ocb_enc_grandloop
				2841
				2842	$L$ocb_enc_short:
				2843	add rdx,6
				2844	jz NEAR $L$ocb_enc_done
				2845
				2846	movdqu xmm2,XMMWORD[rdi]
				2847	cmp rdx,2
				2848	jb NEAR $L$ocb_enc_one
				2849	movdqu xmm3,XMMWORD[16+rdi]
				2850	je NEAR $L$ocb_enc_two
				2851
				2852	movdqu xmm4,XMMWORD[32+rdi]
				2853	cmp rdx,4
				2854	jb NEAR $L$ocb_enc_three
				2855	movdqu xmm5,XMMWORD[48+rdi]
				2856	je NEAR $L$ocb_enc_four
				2857
				2858	movdqu xmm6,XMMWORD[64+rdi]
				2859	pxor xmm7,xmm7
				2860
				2861	call __ocb_encrypt6
				2862
				2863	movdqa xmm15,xmm14
				2864	movups XMMWORD[rsi],xmm2
				2865	movups XMMWORD[16+rsi],xmm3
				2866	movups XMMWORD[32+rsi],xmm4
				2867	movups XMMWORD[48+rsi],xmm5
				2868	movups XMMWORD[64+rsi],xmm6
				2869
				2870	jmp NEAR $L$ocb_enc_done
				2871
				2872	ALIGN 16
				2873	$L$ocb_enc_one:
				2874	movdqa xmm7,xmm10
				2875
				2876	call __ocb_encrypt1
				2877
				2878	movdqa xmm15,xmm7
				2879	movups XMMWORD[rsi],xmm2
				2880	jmp NEAR $L$ocb_enc_done
				2881
				2882	ALIGN 16
				2883	$L$ocb_enc_two:
				2884	pxor xmm4,xmm4
				2885	pxor xmm5,xmm5
				2886
				2887	call __ocb_encrypt4
				2888
				2889	movdqa xmm15,xmm11
				2890	movups XMMWORD[rsi],xmm2
				2891	movups XMMWORD[16+rsi],xmm3
				2892
				2893	jmp NEAR $L$ocb_enc_done
				2894
				2895	ALIGN 16
				2896	$L$ocb_enc_three:
				2897	pxor xmm5,xmm5
				2898
				2899	call __ocb_encrypt4
				2900
				2901	movdqa xmm15,xmm12
				2902	movups XMMWORD[rsi],xmm2
				2903	movups XMMWORD[16+rsi],xmm3
				2904	movups XMMWORD[32+rsi],xmm4
				2905
				2906	jmp NEAR $L$ocb_enc_done
				2907
				2908	ALIGN 16
				2909	$L$ocb_enc_four:
				2910	call __ocb_encrypt4
				2911
				2912	movdqa xmm15,xmm13
				2913	movups XMMWORD[rsi],xmm2
				2914	movups XMMWORD[16+rsi],xmm3
				2915	movups XMMWORD[32+rsi],xmm4
				2916	movups XMMWORD[48+rsi],xmm5
				2917
				2918	$L$ocb_enc_done:
				2919	pxor xmm15,xmm0
				2920	movdqu XMMWORD[rbp],xmm8
				2921	movdqu XMMWORD[r9],xmm15
				2922
				2923	xorps xmm0,xmm0
				2924	pxor xmm1,xmm1
				2925	pxor xmm2,xmm2
				2926	pxor xmm3,xmm3
				2927	pxor xmm4,xmm4
				2928	pxor xmm5,xmm5
				2929	movaps xmm6,XMMWORD[rsp]
				2930	movaps XMMWORD[rsp],xmm0
				2931	movaps xmm7,XMMWORD[16+rsp]
				2932	movaps XMMWORD[16+rsp],xmm0
				2933	movaps xmm8,XMMWORD[32+rsp]
				2934	movaps XMMWORD[32+rsp],xmm0
				2935	movaps xmm9,XMMWORD[48+rsp]
				2936	movaps XMMWORD[48+rsp],xmm0
				2937	movaps xmm10,XMMWORD[64+rsp]
				2938	movaps XMMWORD[64+rsp],xmm0
				2939	movaps xmm11,XMMWORD[80+rsp]
				2940	movaps XMMWORD[80+rsp],xmm0
				2941	movaps xmm12,XMMWORD[96+rsp]
				2942	movaps XMMWORD[96+rsp],xmm0
				2943	movaps xmm13,XMMWORD[112+rsp]
				2944	movaps XMMWORD[112+rsp],xmm0
				2945	movaps xmm14,XMMWORD[128+rsp]
				2946	movaps XMMWORD[128+rsp],xmm0
				2947	movaps xmm15,XMMWORD[144+rsp]
				2948	movaps XMMWORD[144+rsp],xmm0
				2949	lea rax,[((160+40))+rsp]
				2950	$L$ocb_enc_pop:
				2951	mov r14,QWORD[((-40))+rax]
				2952	mov r13,QWORD[((-32))+rax]
				2953	mov r12,QWORD[((-24))+rax]
				2954	mov rbp,QWORD[((-16))+rax]
				2955	mov rbx,QWORD[((-8))+rax]
				2956	lea rsp,[rax]
				2957	$L$ocb_enc_epilogue:
				2958	mov rdi,QWORD[8+rsp] ;WIN64 epilogue
				2959	mov rsi,QWORD[16+rsp]
				2960	DB 0F3h,0C3h ;repret
				2961	$L$SEH_end_aesni_ocb_encrypt:
				2962
				2963
				2964	ALIGN 32
				2965	__ocb_encrypt6:
				2966	pxor xmm15,xmm9
				2967	movdqu xmm11,XMMWORD[r12*1+rbx]
				2968	movdqa xmm12,xmm10
				2969	movdqu xmm13,XMMWORD[r13*1+rbx]
				2970	movdqa xmm14,xmm10
				2971	pxor xmm10,xmm15
				2972	movdqu xmm15,XMMWORD[r14*1+rbx]
				2973	pxor xmm11,xmm10
				2974	pxor xmm8,xmm2
				2975	pxor xmm2,xmm10
				2976	pxor xmm12,xmm11
				2977	pxor xmm8,xmm3
				2978	pxor xmm3,xmm11
				2979	pxor xmm13,xmm12
				2980	pxor xmm8,xmm4
				2981	pxor xmm4,xmm12
				2982	pxor xmm14,xmm13
				2983	pxor xmm8,xmm5
				2984	pxor xmm5,xmm13
				2985	pxor xmm15,xmm14
				2986	pxor xmm8,xmm6
				2987	pxor xmm6,xmm14
				2988	pxor xmm8,xmm7
				2989	pxor xmm7,xmm15
				2990	movups xmm0,XMMWORD[32+r11]
				2991
				2992	lea r12,[1+r8]
				2993	lea r13,[3+r8]
				2994	lea r14,[5+r8]
				2995	add r8,6
				2996	pxor xmm10,xmm9
				2997	bsf r12,r12
				2998	bsf r13,r13
				2999	bsf r14,r14
				3000
				3001	DB 102,15,56,220,209
				3002	DB 102,15,56,220,217
				3003	DB 102,15,56,220,225
				3004	DB 102,15,56,220,233
				3005	pxor xmm11,xmm9
				3006	pxor xmm12,xmm9
				3007	DB 102,15,56,220,241
				3008	pxor xmm13,xmm9
				3009	pxor xmm14,xmm9
				3010	DB 102,15,56,220,249
				3011	movups xmm1,XMMWORD[48+r11]
				3012	pxor xmm15,xmm9
				3013
				3014	DB 102,15,56,220,208
				3015	DB 102,15,56,220,216
				3016	DB 102,15,56,220,224
				3017	DB 102,15,56,220,232
				3018	DB 102,15,56,220,240
				3019	DB 102,15,56,220,248
				3020	movups xmm0,XMMWORD[64+r11]
				3021	shl r12,4
				3022	shl r13,4
				3023	jmp NEAR $L$ocb_enc_loop6
				3024
				3025	ALIGN 32
				3026	$L$ocb_enc_loop6:
				3027	DB 102,15,56,220,209
				3028	DB 102,15,56,220,217
				3029	DB 102,15,56,220,225
				3030	DB 102,15,56,220,233
				3031	DB 102,15,56,220,241
				3032	DB 102,15,56,220,249
				3033	movups xmm1,XMMWORD[rax*1+rcx]
				3034	add rax,32
				3035
				3036	DB 102,15,56,220,208
				3037	DB 102,15,56,220,216
				3038	DB 102,15,56,220,224
				3039	DB 102,15,56,220,232
				3040	DB 102,15,56,220,240
				3041	DB 102,15,56,220,248
				3042	movups xmm0,XMMWORD[((-16))+rax*1+rcx]
				3043	jnz NEAR $L$ocb_enc_loop6
				3044
				3045	DB 102,15,56,220,209
				3046	DB 102,15,56,220,217
				3047	DB 102,15,56,220,225
				3048	DB 102,15,56,220,233
				3049	DB 102,15,56,220,241
				3050	DB 102,15,56,220,249
				3051	movups xmm1,XMMWORD[16+r11]
				3052	shl r14,4
				3053
				3054	DB 102,65,15,56,221,210
				3055	movdqu xmm10,XMMWORD[rbx]
				3056	mov rax,r10
				3057	DB 102,65,15,56,221,219
				3058	DB 102,65,15,56,221,228
				3059	DB 102,65,15,56,221,237
				3060	DB 102,65,15,56,221,246
				3061	DB 102,65,15,56,221,255
				3062	DB 0F3h,0C3h ;repret
				3063
				3064
				3065
				3066	ALIGN 32
				3067	__ocb_encrypt4:
				3068	pxor xmm15,xmm9
				3069	movdqu xmm11,XMMWORD[r12*1+rbx]
				3070	movdqa xmm12,xmm10
				3071	movdqu xmm13,XMMWORD[r13*1+rbx]
				3072	pxor xmm10,xmm15
				3073	pxor xmm11,xmm10
				3074	pxor xmm8,xmm2
				3075	pxor xmm2,xmm10
				3076	pxor xmm12,xmm11
				3077	pxor xmm8,xmm3
				3078	pxor xmm3,xmm11
				3079	pxor xmm13,xmm12
				3080	pxor xmm8,xmm4
				3081	pxor xmm4,xmm12
				3082	pxor xmm8,xmm5
				3083	pxor xmm5,xmm13
				3084	movups xmm0,XMMWORD[32+r11]
				3085
				3086	pxor xmm10,xmm9
				3087	pxor xmm11,xmm9
				3088	pxor xmm12,xmm9
				3089	pxor xmm13,xmm9
				3090
				3091	DB 102,15,56,220,209
				3092	DB 102,15,56,220,217
				3093	DB 102,15,56,220,225
				3094	DB 102,15,56,220,233
				3095	movups xmm1,XMMWORD[48+r11]
				3096
				3097	DB 102,15,56,220,208
				3098	DB 102,15,56,220,216
				3099	DB 102,15,56,220,224
				3100	DB 102,15,56,220,232
				3101	movups xmm0,XMMWORD[64+r11]
				3102	jmp NEAR $L$ocb_enc_loop4
				3103
				3104	ALIGN 32
				3105	$L$ocb_enc_loop4:
				3106	DB 102,15,56,220,209
				3107	DB 102,15,56,220,217
				3108	DB 102,15,56,220,225
				3109	DB 102,15,56,220,233
				3110	movups xmm1,XMMWORD[rax*1+rcx]
				3111	add rax,32
				3112
				3113	DB 102,15,56,220,208
				3114	DB 102,15,56,220,216
				3115	DB 102,15,56,220,224
				3116	DB 102,15,56,220,232
				3117	movups xmm0,XMMWORD[((-16))+rax*1+rcx]
				3118	jnz NEAR $L$ocb_enc_loop4
				3119
				3120	DB 102,15,56,220,209
				3121	DB 102,15,56,220,217
				3122	DB 102,15,56,220,225
				3123	DB 102,15,56,220,233
				3124	movups xmm1,XMMWORD[16+r11]
				3125	mov rax,r10
				3126
				3127	DB 102,65,15,56,221,210
				3128	DB 102,65,15,56,221,219
				3129	DB 102,65,15,56,221,228
				3130	DB 102,65,15,56,221,237
				3131	DB 0F3h,0C3h ;repret
				3132
				3133
				3134
				3135	ALIGN 32
				3136	__ocb_encrypt1:
				3137	pxor xmm7,xmm15
				3138	pxor xmm7,xmm9
				3139	pxor xmm8,xmm2
				3140	pxor xmm2,xmm7
				3141	movups xmm0,XMMWORD[32+r11]
				3142
				3143	DB 102,15,56,220,209
				3144	movups xmm1,XMMWORD[48+r11]
				3145	pxor xmm7,xmm9
				3146
				3147	DB 102,15,56,220,208
				3148	movups xmm0,XMMWORD[64+r11]
				3149	jmp NEAR $L$ocb_enc_loop1
				3150
				3151	ALIGN 32
				3152	$L$ocb_enc_loop1:
				3153	DB 102,15,56,220,209
				3154	movups xmm1,XMMWORD[rax*1+rcx]
				3155	add rax,32
				3156
				3157	DB 102,15,56,220,208
				3158	movups xmm0,XMMWORD[((-16))+rax*1+rcx]
				3159	jnz NEAR $L$ocb_enc_loop1
				3160
				3161	DB 102,15,56,220,209
				3162	movups xmm1,XMMWORD[16+r11]
				3163	mov rax,r10
				3164
				3165	DB 102,15,56,221,215
				3166	DB 0F3h,0C3h ;repret
				3167
				3168
				3169	global aesni_ocb_decrypt
				3170
				3171	ALIGN 32
				3172	aesni_ocb_decrypt:
				3173	mov QWORD[8+rsp],rdi ;WIN64 prologue
				3174	mov QWORD[16+rsp],rsi
				3175	mov rax,rsp
				3176	$L$SEH_begin_aesni_ocb_decrypt:
				3177	mov rdi,rcx
				3178	mov rsi,rdx
				3179	mov rdx,r8
				3180	mov rcx,r9
				3181	mov r8,QWORD[40+rsp]
				3182	mov r9,QWORD[48+rsp]
				3183
				3184
				3185	lea rax,[rsp]
				3186	push rbx
				3187	push rbp
				3188	push r12
				3189	push r13
				3190	push r14
				3191	lea rsp,[((-160))+rsp]
				3192	movaps XMMWORD[rsp],xmm6
				3193	movaps XMMWORD[16+rsp],xmm7
				3194	movaps XMMWORD[32+rsp],xmm8
				3195	movaps XMMWORD[48+rsp],xmm9
				3196	movaps XMMWORD[64+rsp],xmm10
				3197	movaps XMMWORD[80+rsp],xmm11
				3198	movaps XMMWORD[96+rsp],xmm12
				3199	movaps XMMWORD[112+rsp],xmm13
				3200	movaps XMMWORD[128+rsp],xmm14
				3201	movaps XMMWORD[144+rsp],xmm15
				3202	$L$ocb_dec_body:
				3203	mov rbx,QWORD[56+rax]
				3204	mov rbp,QWORD[((56+8))+rax]
				3205
				3206	mov r10d,DWORD[240+rcx]
				3207	mov r11,rcx
				3208	shl r10d,4
				3209	movups xmm9,XMMWORD[rcx]
				3210	movups xmm1,XMMWORD[16+r10*1+rcx]
				3211
				3212	movdqu xmm15,XMMWORD[r9]
				3213	pxor xmm9,xmm1
				3214	pxor xmm15,xmm1
				3215
				3216	mov eax,16+32
				3217	lea rcx,[32+r10*1+r11]
				3218	movups xmm1,XMMWORD[16+r11]
				3219	sub rax,r10
				3220	mov r10,rax
				3221
				3222	movdqu xmm10,XMMWORD[rbx]
				3223	movdqu xmm8,XMMWORD[rbp]
				3224
				3225	test r8,1
				3226	jnz NEAR $L$ocb_dec_odd
				3227
				3228	bsf r12,r8
				3229	add r8,1
				3230	shl r12,4
				3231	movdqu xmm7,XMMWORD[r12*1+rbx]
				3232	movdqu xmm2,XMMWORD[rdi]
				3233	lea rdi,[16+rdi]
				3234
				3235	call __ocb_decrypt1
				3236
				3237	movdqa xmm15,xmm7
				3238	movups XMMWORD[rsi],xmm2
				3239	xorps xmm8,xmm2
				3240	lea rsi,[16+rsi]
				3241	sub rdx,1
				3242	jz NEAR $L$ocb_dec_done
				3243
				3244	$L$ocb_dec_odd:
				3245	lea r12,[1+r8]
				3246	lea r13,[3+r8]
				3247	lea r14,[5+r8]
				3248	lea r8,[6+r8]
				3249	bsf r12,r12
				3250	bsf r13,r13
				3251	bsf r14,r14
				3252	shl r12,4
				3253	shl r13,4
				3254	shl r14,4
				3255
				3256	sub rdx,6
				3257	jc NEAR $L$ocb_dec_short
				3258	jmp NEAR $L$ocb_dec_grandloop
				3259
				3260	ALIGN 32
				3261	$L$ocb_dec_grandloop:
				3262	movdqu xmm2,XMMWORD[rdi]
				3263	movdqu xmm3,XMMWORD[16+rdi]
				3264	movdqu xmm4,XMMWORD[32+rdi]
				3265	movdqu xmm5,XMMWORD[48+rdi]
				3266	movdqu xmm6,XMMWORD[64+rdi]
				3267	movdqu xmm7,XMMWORD[80+rdi]
				3268	lea rdi,[96+rdi]
				3269
				3270	call __ocb_decrypt6
				3271
				3272	movups XMMWORD[rsi],xmm2
				3273	pxor xmm8,xmm2
				3274	movups XMMWORD[16+rsi],xmm3
				3275	pxor xmm8,xmm3
				3276	movups XMMWORD[32+rsi],xmm4
				3277	pxor xmm8,xmm4
				3278	movups XMMWORD[48+rsi],xmm5
				3279	pxor xmm8,xmm5
				3280	movups XMMWORD[64+rsi],xmm6
				3281	pxor xmm8,xmm6
				3282	movups XMMWORD[80+rsi],xmm7
				3283	pxor xmm8,xmm7
				3284	lea rsi,[96+rsi]
				3285	sub rdx,6
				3286	jnc NEAR $L$ocb_dec_grandloop
				3287
				3288	$L$ocb_dec_short:
				3289	add rdx,6
				3290	jz NEAR $L$ocb_dec_done
				3291
				3292	movdqu xmm2,XMMWORD[rdi]
				3293	cmp rdx,2
				3294	jb NEAR $L$ocb_dec_one
				3295	movdqu xmm3,XMMWORD[16+rdi]
				3296	je NEAR $L$ocb_dec_two
				3297
				3298	movdqu xmm4,XMMWORD[32+rdi]
				3299	cmp rdx,4
				3300	jb NEAR $L$ocb_dec_three
				3301	movdqu xmm5,XMMWORD[48+rdi]
				3302	je NEAR $L$ocb_dec_four
				3303
				3304	movdqu xmm6,XMMWORD[64+rdi]
				3305	pxor xmm7,xmm7
				3306
				3307	call __ocb_decrypt6
				3308
				3309	movdqa xmm15,xmm14
				3310	movups XMMWORD[rsi],xmm2
				3311	pxor xmm8,xmm2
				3312	movups XMMWORD[16+rsi],xmm3
				3313	pxor xmm8,xmm3
				3314	movups XMMWORD[32+rsi],xmm4
				3315	pxor xmm8,xmm4
				3316	movups XMMWORD[48+rsi],xmm5
				3317	pxor xmm8,xmm5
				3318	movups XMMWORD[64+rsi],xmm6
				3319	pxor xmm8,xmm6
				3320
				3321	jmp NEAR $L$ocb_dec_done
				3322
				3323	ALIGN 16
				3324	$L$ocb_dec_one:
				3325	movdqa xmm7,xmm10
				3326
				3327	call __ocb_decrypt1
				3328
				3329	movdqa xmm15,xmm7
				3330	movups XMMWORD[rsi],xmm2
				3331	xorps xmm8,xmm2
				3332	jmp NEAR $L$ocb_dec_done
				3333
				3334	ALIGN 16
				3335	$L$ocb_dec_two:
				3336	pxor xmm4,xmm4
				3337	pxor xmm5,xmm5
				3338
				3339	call __ocb_decrypt4
				3340
				3341	movdqa xmm15,xmm11
				3342	movups XMMWORD[rsi],xmm2
				3343	xorps xmm8,xmm2
				3344	movups XMMWORD[16+rsi],xmm3
				3345	xorps xmm8,xmm3
				3346
				3347	jmp NEAR $L$ocb_dec_done
				3348
				3349	ALIGN 16
				3350	$L$ocb_dec_three:
				3351	pxor xmm5,xmm5
				3352
				3353	call __ocb_decrypt4
				3354
				3355	movdqa xmm15,xmm12
				3356	movups XMMWORD[rsi],xmm2
				3357	xorps xmm8,xmm2
				3358	movups XMMWORD[16+rsi],xmm3
				3359	xorps xmm8,xmm3
				3360	movups XMMWORD[32+rsi],xmm4
				3361	xorps xmm8,xmm4
				3362
				3363	jmp NEAR $L$ocb_dec_done
				3364
				3365	ALIGN 16
				3366	$L$ocb_dec_four:
				3367	call __ocb_decrypt4
				3368
				3369	movdqa xmm15,xmm13
				3370	movups XMMWORD[rsi],xmm2
				3371	pxor xmm8,xmm2
				3372	movups XMMWORD[16+rsi],xmm3
				3373	pxor xmm8,xmm3
				3374	movups XMMWORD[32+rsi],xmm4
				3375	pxor xmm8,xmm4
				3376	movups XMMWORD[48+rsi],xmm5
				3377	pxor xmm8,xmm5
				3378
				3379	$L$ocb_dec_done:
				3380	pxor xmm15,xmm0
				3381	movdqu XMMWORD[rbp],xmm8
				3382	movdqu XMMWORD[r9],xmm15
				3383
				3384	xorps xmm0,xmm0
				3385	pxor xmm1,xmm1
				3386	pxor xmm2,xmm2
				3387	pxor xmm3,xmm3
				3388	pxor xmm4,xmm4
				3389	pxor xmm5,xmm5
				3390	movaps xmm6,XMMWORD[rsp]
				3391	movaps XMMWORD[rsp],xmm0
				3392	movaps xmm7,XMMWORD[16+rsp]
				3393	movaps XMMWORD[16+rsp],xmm0
				3394	movaps xmm8,XMMWORD[32+rsp]
				3395	movaps XMMWORD[32+rsp],xmm0
				3396	movaps xmm9,XMMWORD[48+rsp]
				3397	movaps XMMWORD[48+rsp],xmm0
				3398	movaps xmm10,XMMWORD[64+rsp]
				3399	movaps XMMWORD[64+rsp],xmm0
				3400	movaps xmm11,XMMWORD[80+rsp]
				3401	movaps XMMWORD[80+rsp],xmm0
				3402	movaps xmm12,XMMWORD[96+rsp]
				3403	movaps XMMWORD[96+rsp],xmm0
				3404	movaps xmm13,XMMWORD[112+rsp]
				3405	movaps XMMWORD[112+rsp],xmm0
				3406	movaps xmm14,XMMWORD[128+rsp]
				3407	movaps XMMWORD[128+rsp],xmm0
				3408	movaps xmm15,XMMWORD[144+rsp]
				3409	movaps XMMWORD[144+rsp],xmm0
				3410	lea rax,[((160+40))+rsp]
				3411	$L$ocb_dec_pop:
				3412	mov r14,QWORD[((-40))+rax]
				3413	mov r13,QWORD[((-32))+rax]
				3414	mov r12,QWORD[((-24))+rax]
				3415	mov rbp,QWORD[((-16))+rax]
				3416	mov rbx,QWORD[((-8))+rax]
				3417	lea rsp,[rax]
				3418	$L$ocb_dec_epilogue:
				3419	mov rdi,QWORD[8+rsp] ;WIN64 epilogue
				3420	mov rsi,QWORD[16+rsp]
				3421	DB 0F3h,0C3h ;repret
				3422	$L$SEH_end_aesni_ocb_decrypt:
				3423
				3424
				3425	ALIGN 32
				3426	__ocb_decrypt6:
				3427	pxor xmm15,xmm9
				3428	movdqu xmm11,XMMWORD[r12*1+rbx]
				3429	movdqa xmm12,xmm10
				3430	movdqu xmm13,XMMWORD[r13*1+rbx]
				3431	movdqa xmm14,xmm10
				3432	pxor xmm10,xmm15
				3433	movdqu xmm15,XMMWORD[r14*1+rbx]
				3434	pxor xmm11,xmm10
				3435	pxor xmm2,xmm10
				3436	pxor xmm12,xmm11
				3437	pxor xmm3,xmm11
				3438	pxor xmm13,xmm12
				3439	pxor xmm4,xmm12
				3440	pxor xmm14,xmm13
				3441	pxor xmm5,xmm13
				3442	pxor xmm15,xmm14
				3443	pxor xmm6,xmm14
				3444	pxor xmm7,xmm15
				3445	movups xmm0,XMMWORD[32+r11]
				3446
				3447	lea r12,[1+r8]
				3448	lea r13,[3+r8]
				3449	lea r14,[5+r8]
				3450	add r8,6
				3451	pxor xmm10,xmm9
				3452	bsf r12,r12
				3453	bsf r13,r13
				3454	bsf r14,r14
				3455
				3456	DB 102,15,56,222,209
				3457	DB 102,15,56,222,217
				3458	DB 102,15,56,222,225
				3459	DB 102,15,56,222,233
				3460	pxor xmm11,xmm9
				3461	pxor xmm12,xmm9
				3462	DB 102,15,56,222,241
				3463	pxor xmm13,xmm9
				3464	pxor xmm14,xmm9
				3465	DB 102,15,56,222,249
				3466	movups xmm1,XMMWORD[48+r11]
				3467	pxor xmm15,xmm9
				3468
				3469	DB 102,15,56,222,208
				3470	DB 102,15,56,222,216
				3471	DB 102,15,56,222,224
				3472	DB 102,15,56,222,232
				3473	DB 102,15,56,222,240
				3474	DB 102,15,56,222,248
				3475	movups xmm0,XMMWORD[64+r11]
				3476	shl r12,4
				3477	shl r13,4
				3478	jmp NEAR $L$ocb_dec_loop6
				3479
				3480	ALIGN 32
				3481	$L$ocb_dec_loop6:
				3482	DB 102,15,56,222,209
				3483	DB 102,15,56,222,217
				3484	DB 102,15,56,222,225
				3485	DB 102,15,56,222,233
				3486	DB 102,15,56,222,241
				3487	DB 102,15,56,222,249
				3488	movups xmm1,XMMWORD[rax*1+rcx]
				3489	add rax,32
				3490
				3491	DB 102,15,56,222,208
				3492	DB 102,15,56,222,216
				3493	DB 102,15,56,222,224
				3494	DB 102,15,56,222,232
				3495	DB 102,15,56,222,240
				3496	DB 102,15,56,222,248
				3497	movups xmm0,XMMWORD[((-16))+rax*1+rcx]
				3498	jnz NEAR $L$ocb_dec_loop6
				3499
				3500	DB 102,15,56,222,209
				3501	DB 102,15,56,222,217
				3502	DB 102,15,56,222,225
				3503	DB 102,15,56,222,233
				3504	DB 102,15,56,222,241
				3505	DB 102,15,56,222,249
				3506	movups xmm1,XMMWORD[16+r11]
				3507	shl r14,4
				3508
				3509	DB 102,65,15,56,223,210
				3510	movdqu xmm10,XMMWORD[rbx]
				3511	mov rax,r10
				3512	DB 102,65,15,56,223,219
				3513	DB 102,65,15,56,223,228
				3514	DB 102,65,15,56,223,237
				3515	DB 102,65,15,56,223,246
				3516	DB 102,65,15,56,223,255
				3517	DB 0F3h,0C3h ;repret
				3518
				3519
				3520
				3521	ALIGN 32
				3522	__ocb_decrypt4:
				3523	pxor xmm15,xmm9
				3524	movdqu xmm11,XMMWORD[r12*1+rbx]
				3525	movdqa xmm12,xmm10
				3526	movdqu xmm13,XMMWORD[r13*1+rbx]
				3527	pxor xmm10,xmm15
				3528	pxor xmm11,xmm10
				3529	pxor xmm2,xmm10
				3530	pxor xmm12,xmm11
				3531	pxor xmm3,xmm11
				3532	pxor xmm13,xmm12
				3533	pxor xmm4,xmm12
				3534	pxor xmm5,xmm13
				3535	movups xmm0,XMMWORD[32+r11]
				3536
				3537	pxor xmm10,xmm9
				3538	pxor xmm11,xmm9
				3539	pxor xmm12,xmm9
				3540	pxor xmm13,xmm9
				3541
				3542	DB 102,15,56,222,209
				3543	DB 102,15,56,222,217
				3544	DB 102,15,56,222,225
				3545	DB 102,15,56,222,233
				3546	movups xmm1,XMMWORD[48+r11]
				3547
				3548	DB 102,15,56,222,208
				3549	DB 102,15,56,222,216
				3550	DB 102,15,56,222,224
				3551	DB 102,15,56,222,232
				3552	movups xmm0,XMMWORD[64+r11]
				3553	jmp NEAR $L$ocb_dec_loop4
				3554
				3555	ALIGN 32
				3556	$L$ocb_dec_loop4:
				3557	DB 102,15,56,222,209
				3558	DB 102,15,56,222,217
				3559	DB 102,15,56,222,225
				3560	DB 102,15,56,222,233
				3561	movups xmm1,XMMWORD[rax*1+rcx]
				3562	add rax,32
				3563
				3564	DB 102,15,56,222,208
				3565	DB 102,15,56,222,216
				3566	DB 102,15,56,222,224
				3567	DB 102,15,56,222,232
				3568	movups xmm0,XMMWORD[((-16))+rax*1+rcx]
				3569	jnz NEAR $L$ocb_dec_loop4
				3570
				3571	DB 102,15,56,222,209
				3572	DB 102,15,56,222,217
				3573	DB 102,15,56,222,225
				3574	DB 102,15,56,222,233
				3575	movups xmm1,XMMWORD[16+r11]
				3576	mov rax,r10
				3577
				3578	DB 102,65,15,56,223,210
				3579	DB 102,65,15,56,223,219
				3580	DB 102,65,15,56,223,228
				3581	DB 102,65,15,56,223,237
				3582	DB 0F3h,0C3h ;repret
				3583
				3584
				3585
				3586	ALIGN 32
				3587	__ocb_decrypt1:
				3588	pxor xmm7,xmm15
				3589	pxor xmm7,xmm9
				3590	pxor xmm2,xmm7
				3591	movups xmm0,XMMWORD[32+r11]
				3592
				3593	DB 102,15,56,222,209
				3594	movups xmm1,XMMWORD[48+r11]
				3595	pxor xmm7,xmm9
				3596
				3597	DB 102,15,56,222,208
				3598	movups xmm0,XMMWORD[64+r11]
				3599	jmp NEAR $L$ocb_dec_loop1
				3600
				3601	ALIGN 32
				3602	$L$ocb_dec_loop1:
				3603	DB 102,15,56,222,209
				3604	movups xmm1,XMMWORD[rax*1+rcx]
				3605	add rax,32
				3606
				3607	DB 102,15,56,222,208
				3608	movups xmm0,XMMWORD[((-16))+rax*1+rcx]
				3609	jnz NEAR $L$ocb_dec_loop1
				3610
				3611	DB 102,15,56,222,209
				3612	movups xmm1,XMMWORD[16+r11]
				3613	mov rax,r10
				3614
				3615	DB 102,15,56,223,215
				3616	DB 0F3h,0C3h ;repret
				3617
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3618	global aesni_cbc_encrypt
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3619
				3620	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3621	aesni_cbc_encrypt:
				3622	mov QWORD[8+rsp],rdi ;WIN64 prologue
				3623	mov QWORD[16+rsp],rsi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3624	mov rax,rsp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3625	$L$SEH_begin_aesni_cbc_encrypt:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3626	mov rdi,rcx
				3627	mov rsi,rdx
				3628	mov rdx,r8
				3629	mov rcx,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3630	mov r8,QWORD[40+rsp]
				3631	mov r9,QWORD[48+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3632
				3633
				3634	test rdx,rdx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3635	jz NEAR $L$cbc_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3636
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3637	mov r10d,DWORD[240+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3638	mov r11,rcx
				3639	test r9d,r9d
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3640	jz NEAR $L$cbc_decrypt
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3641
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3642	movups xmm2,XMMWORD[r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3643	mov eax,r10d
				3644	cmp rdx,16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3645	jb NEAR $L$cbc_enc_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3646	sub rdx,16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3647	jmp NEAR $L$cbc_enc_loop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3648	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3649	$L$cbc_enc_loop:
				3650	movups xmm3,XMMWORD[rdi]
				3651	lea rdi,[16+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3652
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3653	movups xmm0,XMMWORD[rcx]
				3654	movups xmm1,XMMWORD[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3655	xorps xmm3,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3656	lea rcx,[32+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3657	xorps xmm2,xmm3
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3658	$L$oop_enc1_15:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3659	DB 102,15,56,220,209
				3660	dec eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3661	movups xmm1,XMMWORD[rcx]
				3662	lea rcx,[16+rcx]
				3663	jnz NEAR $L$oop_enc1_15
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3664	DB 102,15,56,221,209
				3665	mov eax,r10d
				3666	mov rcx,r11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3667	movups XMMWORD[rsi],xmm2
				3668	lea rsi,[16+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3669	sub rdx,16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3670	jnc NEAR $L$cbc_enc_loop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3671	add rdx,16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3672	jnz NEAR $L$cbc_enc_tail
				3673	pxor xmm0,xmm0
				3674	pxor xmm1,xmm1
				3675	movups XMMWORD[r8],xmm2
				3676	pxor xmm2,xmm2
				3677	pxor xmm3,xmm3
				3678	jmp NEAR $L$cbc_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3679
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3680	$L$cbc_enc_tail:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3681	mov rcx,rdx
				3682	xchg rsi,rdi
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3683	DD 0x9066A4F3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3684	mov ecx,16
				3685	sub rcx,rdx
				3686	xor eax,eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3687	DD 0x9066AAF3
				3688	lea rdi,[((-16))+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3689	mov eax,r10d
				3690	mov rsi,rdi
				3691	mov rcx,r11
				3692	xor rdx,rdx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3693	jmp NEAR $L$cbc_enc_loop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3694
				3695	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3696	$L$cbc_decrypt:
				3697	cmp rdx,16
				3698	jne NEAR $L$cbc_decrypt_bulk
				3699
				3700
				3701
				3702	movdqu xmm2,XMMWORD[rdi]
				3703	movdqu xmm3,XMMWORD[r8]
				3704	movdqa xmm4,xmm2
				3705	movups xmm0,XMMWORD[rcx]
				3706	movups xmm1,XMMWORD[16+rcx]
				3707	lea rcx,[32+rcx]
				3708	xorps xmm2,xmm0
				3709	$L$oop_dec1_16:
				3710	DB 102,15,56,222,209
				3711	dec r10d
				3712	movups xmm1,XMMWORD[rcx]
				3713	lea rcx,[16+rcx]
				3714	jnz NEAR $L$oop_dec1_16
				3715	DB 102,15,56,223,209
				3716	pxor xmm0,xmm0
				3717	pxor xmm1,xmm1
				3718	movdqu XMMWORD[r8],xmm4
				3719	xorps xmm2,xmm3
				3720	pxor xmm3,xmm3
				3721	movups XMMWORD[rsi],xmm2
				3722	pxor xmm2,xmm2
				3723	jmp NEAR $L$cbc_ret
				3724	ALIGN 16
				3725	$L$cbc_decrypt_bulk:
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	3726	lea r11,[rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3727	push rbp
				3728	sub rsp,176
				3729	and rsp,-16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3730	movaps XMMWORD[16+rsp],xmm6
				3731	movaps XMMWORD[32+rsp],xmm7
				3732	movaps XMMWORD[48+rsp],xmm8
				3733	movaps XMMWORD[64+rsp],xmm9
				3734	movaps XMMWORD[80+rsp],xmm10
				3735	movaps XMMWORD[96+rsp],xmm11
				3736	movaps XMMWORD[112+rsp],xmm12
				3737	movaps XMMWORD[128+rsp],xmm13
				3738	movaps XMMWORD[144+rsp],xmm14
				3739	movaps XMMWORD[160+rsp],xmm15
				3740	$L$cbc_decrypt_body:
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	3741	mov rbp,rcx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3742	movups xmm10,XMMWORD[r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3743	mov eax,r10d
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3744	cmp rdx,0x50
				3745	jbe NEAR $L$cbc_dec_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3746
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3747	movups xmm0,XMMWORD[rcx]
				3748	movdqu xmm2,XMMWORD[rdi]
				3749	movdqu xmm3,XMMWORD[16+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3750	movdqa xmm11,xmm2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3751	movdqu xmm4,XMMWORD[32+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3752	movdqa xmm12,xmm3
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3753	movdqu xmm5,XMMWORD[48+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3754	movdqa xmm13,xmm4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3755	movdqu xmm6,XMMWORD[64+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3756	movdqa xmm14,xmm5
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3757	movdqu xmm7,XMMWORD[80+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3758	movdqa xmm15,xmm6
Robert Sloan	2424d84	2017-05-01 07:46:28 -0700	[diff] [blame]	3759	lea r9,[OPENSSL_ia32cap_P]
Robert Sloan	572a4e2	2017-04-17 10:52:19 -0700	[diff] [blame]	3760	mov r9d,DWORD[4+r9]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3761	cmp rdx,0x70
				3762	jbe NEAR $L$cbc_dec_six_or_seven
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3763
				3764	and r9d,71303168
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3765	sub rdx,0x50
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3766	cmp r9d,4194304
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3767	je NEAR $L$cbc_dec_loop6_enter
				3768	sub rdx,0x20
				3769	lea rcx,[112+rcx]
				3770	jmp NEAR $L$cbc_dec_loop8_enter
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3771	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3772	$L$cbc_dec_loop8:
				3773	movups XMMWORD[rsi],xmm9
				3774	lea rsi,[16+rsi]
				3775	$L$cbc_dec_loop8_enter:
				3776	movdqu xmm8,XMMWORD[96+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3777	pxor xmm2,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3778	movdqu xmm9,XMMWORD[112+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3779	pxor xmm3,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3780	movups xmm1,XMMWORD[((16-112))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3781	pxor xmm4,xmm0
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	3782	mov rbp,-1
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3783	cmp rdx,0x70
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3784	pxor xmm5,xmm0
				3785	pxor xmm6,xmm0
				3786	pxor xmm7,xmm0
				3787	pxor xmm8,xmm0
				3788
				3789	DB 102,15,56,222,209
				3790	pxor xmm9,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3791	movups xmm0,XMMWORD[((32-112))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3792	DB 102,15,56,222,217
				3793	DB 102,15,56,222,225
				3794	DB 102,15,56,222,233
				3795	DB 102,15,56,222,241
				3796	DB 102,15,56,222,249
				3797	DB 102,68,15,56,222,193
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	3798	adc rbp,0
				3799	and rbp,128
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3800	DB 102,68,15,56,222,201
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	3801	add rbp,rdi
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3802	movups xmm1,XMMWORD[((48-112))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3803	DB 102,15,56,222,208
				3804	DB 102,15,56,222,216
				3805	DB 102,15,56,222,224
				3806	DB 102,15,56,222,232
				3807	DB 102,15,56,222,240
				3808	DB 102,15,56,222,248
				3809	DB 102,68,15,56,222,192
				3810	DB 102,68,15,56,222,200
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3811	movups xmm0,XMMWORD[((64-112))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3812	nop
				3813	DB 102,15,56,222,209
				3814	DB 102,15,56,222,217
				3815	DB 102,15,56,222,225
				3816	DB 102,15,56,222,233
				3817	DB 102,15,56,222,241
				3818	DB 102,15,56,222,249
				3819	DB 102,68,15,56,222,193
				3820	DB 102,68,15,56,222,201
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3821	movups xmm1,XMMWORD[((80-112))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3822	nop
				3823	DB 102,15,56,222,208
				3824	DB 102,15,56,222,216
				3825	DB 102,15,56,222,224
				3826	DB 102,15,56,222,232
				3827	DB 102,15,56,222,240
				3828	DB 102,15,56,222,248
				3829	DB 102,68,15,56,222,192
				3830	DB 102,68,15,56,222,200
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3831	movups xmm0,XMMWORD[((96-112))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3832	nop
				3833	DB 102,15,56,222,209
				3834	DB 102,15,56,222,217
				3835	DB 102,15,56,222,225
				3836	DB 102,15,56,222,233
				3837	DB 102,15,56,222,241
				3838	DB 102,15,56,222,249
				3839	DB 102,68,15,56,222,193
				3840	DB 102,68,15,56,222,201
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3841	movups xmm1,XMMWORD[((112-112))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3842	nop
				3843	DB 102,15,56,222,208
				3844	DB 102,15,56,222,216
				3845	DB 102,15,56,222,224
				3846	DB 102,15,56,222,232
				3847	DB 102,15,56,222,240
				3848	DB 102,15,56,222,248
				3849	DB 102,68,15,56,222,192
				3850	DB 102,68,15,56,222,200
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3851	movups xmm0,XMMWORD[((128-112))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3852	nop
				3853	DB 102,15,56,222,209
				3854	DB 102,15,56,222,217
				3855	DB 102,15,56,222,225
				3856	DB 102,15,56,222,233
				3857	DB 102,15,56,222,241
				3858	DB 102,15,56,222,249
				3859	DB 102,68,15,56,222,193
				3860	DB 102,68,15,56,222,201
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3861	movups xmm1,XMMWORD[((144-112))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3862	cmp eax,11
				3863	DB 102,15,56,222,208
				3864	DB 102,15,56,222,216
				3865	DB 102,15,56,222,224
				3866	DB 102,15,56,222,232
				3867	DB 102,15,56,222,240
				3868	DB 102,15,56,222,248
				3869	DB 102,68,15,56,222,192
				3870	DB 102,68,15,56,222,200
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3871	movups xmm0,XMMWORD[((160-112))+rcx]
				3872	jb NEAR $L$cbc_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3873	DB 102,15,56,222,209
				3874	DB 102,15,56,222,217
				3875	DB 102,15,56,222,225
				3876	DB 102,15,56,222,233
				3877	DB 102,15,56,222,241
				3878	DB 102,15,56,222,249
				3879	DB 102,68,15,56,222,193
				3880	DB 102,68,15,56,222,201
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3881	movups xmm1,XMMWORD[((176-112))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3882	nop
				3883	DB 102,15,56,222,208
				3884	DB 102,15,56,222,216
				3885	DB 102,15,56,222,224
				3886	DB 102,15,56,222,232
				3887	DB 102,15,56,222,240
				3888	DB 102,15,56,222,248
				3889	DB 102,68,15,56,222,192
				3890	DB 102,68,15,56,222,200
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3891	movups xmm0,XMMWORD[((192-112))+rcx]
				3892	je NEAR $L$cbc_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3893	DB 102,15,56,222,209
				3894	DB 102,15,56,222,217
				3895	DB 102,15,56,222,225
				3896	DB 102,15,56,222,233
				3897	DB 102,15,56,222,241
				3898	DB 102,15,56,222,249
				3899	DB 102,68,15,56,222,193
				3900	DB 102,68,15,56,222,201
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3901	movups xmm1,XMMWORD[((208-112))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3902	nop
				3903	DB 102,15,56,222,208
				3904	DB 102,15,56,222,216
				3905	DB 102,15,56,222,224
				3906	DB 102,15,56,222,232
				3907	DB 102,15,56,222,240
				3908	DB 102,15,56,222,248
				3909	DB 102,68,15,56,222,192
				3910	DB 102,68,15,56,222,200
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3911	movups xmm0,XMMWORD[((224-112))+rcx]
				3912	jmp NEAR $L$cbc_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3913	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3914	$L$cbc_dec_done:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3915	DB 102,15,56,222,209
				3916	DB 102,15,56,222,217
				3917	pxor xmm10,xmm0
				3918	pxor xmm11,xmm0
				3919	DB 102,15,56,222,225
				3920	DB 102,15,56,222,233
				3921	pxor xmm12,xmm0
				3922	pxor xmm13,xmm0
				3923	DB 102,15,56,222,241
				3924	DB 102,15,56,222,249
				3925	pxor xmm14,xmm0
				3926	pxor xmm15,xmm0
				3927	DB 102,68,15,56,222,193
				3928	DB 102,68,15,56,222,201
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3929	movdqu xmm1,XMMWORD[80+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3930
				3931	DB 102,65,15,56,223,210
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3932	movdqu xmm10,XMMWORD[96+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3933	pxor xmm1,xmm0
				3934	DB 102,65,15,56,223,219
				3935	pxor xmm10,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3936	movdqu xmm0,XMMWORD[112+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3937	DB 102,65,15,56,223,228
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3938	lea rdi,[128+rdi]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	3939	movdqu xmm11,XMMWORD[rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3940	DB 102,65,15,56,223,237
				3941	DB 102,65,15,56,223,246
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	3942	movdqu xmm12,XMMWORD[16+rbp]
				3943	movdqu xmm13,XMMWORD[32+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3944	DB 102,65,15,56,223,255
				3945	DB 102,68,15,56,223,193
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	3946	movdqu xmm14,XMMWORD[48+rbp]
				3947	movdqu xmm15,XMMWORD[64+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3948	DB 102,69,15,56,223,202
				3949	movdqa xmm10,xmm0
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	3950	movdqu xmm1,XMMWORD[80+rbp]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3951	movups xmm0,XMMWORD[((-112))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3952
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3953	movups XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3954	movdqa xmm2,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3955	movups XMMWORD[16+rsi],xmm3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3956	movdqa xmm3,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3957	movups XMMWORD[32+rsi],xmm4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3958	movdqa xmm4,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3959	movups XMMWORD[48+rsi],xmm5
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3960	movdqa xmm5,xmm14
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3961	movups XMMWORD[64+rsi],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3962	movdqa xmm6,xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3963	movups XMMWORD[80+rsi],xmm7
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3964	movdqa xmm7,xmm1
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3965	movups XMMWORD[96+rsi],xmm8
				3966	lea rsi,[112+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3967
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3968	sub rdx,0x80
				3969	ja NEAR $L$cbc_dec_loop8
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3970
				3971	movaps xmm2,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3972	lea rcx,[((-112))+rcx]
				3973	add rdx,0x70
				3974	jle NEAR $L$cbc_dec_clear_tail_collected
				3975	movups XMMWORD[rsi],xmm9
				3976	lea rsi,[16+rsi]
				3977	cmp rdx,0x50
				3978	jbe NEAR $L$cbc_dec_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3979
				3980	movaps xmm2,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3981	$L$cbc_dec_six_or_seven:
				3982	cmp rdx,0x60
				3983	ja NEAR $L$cbc_dec_seven
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3984
				3985	movaps xmm8,xmm7
				3986	call _aesni_decrypt6
				3987	pxor xmm2,xmm10
				3988	movaps xmm10,xmm8
				3989	pxor xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3990	movdqu XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3991	pxor xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3992	movdqu XMMWORD[16+rsi],xmm3
				3993	pxor xmm3,xmm3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3994	pxor xmm5,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3995	movdqu XMMWORD[32+rsi],xmm4
				3996	pxor xmm4,xmm4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	3997	pxor xmm6,xmm14
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	3998	movdqu XMMWORD[48+rsi],xmm5
				3999	pxor xmm5,xmm5
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4000	pxor xmm7,xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4001	movdqu XMMWORD[64+rsi],xmm6
				4002	pxor xmm6,xmm6
				4003	lea rsi,[80+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4004	movdqa xmm2,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4005	pxor xmm7,xmm7
				4006	jmp NEAR $L$cbc_dec_tail_collected
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4007
				4008	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4009	$L$cbc_dec_seven:
				4010	movups xmm8,XMMWORD[96+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4011	xorps xmm9,xmm9
				4012	call _aesni_decrypt8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4013	movups xmm9,XMMWORD[80+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4014	pxor xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4015	movups xmm10,XMMWORD[96+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4016	pxor xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4017	movdqu XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4018	pxor xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4019	movdqu XMMWORD[16+rsi],xmm3
				4020	pxor xmm3,xmm3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4021	pxor xmm5,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4022	movdqu XMMWORD[32+rsi],xmm4
				4023	pxor xmm4,xmm4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4024	pxor xmm6,xmm14
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4025	movdqu XMMWORD[48+rsi],xmm5
				4026	pxor xmm5,xmm5
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4027	pxor xmm7,xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4028	movdqu XMMWORD[64+rsi],xmm6
				4029	pxor xmm6,xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4030	pxor xmm8,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4031	movdqu XMMWORD[80+rsi],xmm7
				4032	pxor xmm7,xmm7
				4033	lea rsi,[96+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4034	movdqa xmm2,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4035	pxor xmm8,xmm8
				4036	pxor xmm9,xmm9
				4037	jmp NEAR $L$cbc_dec_tail_collected
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4038
				4039	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4040	$L$cbc_dec_loop6:
				4041	movups XMMWORD[rsi],xmm7
				4042	lea rsi,[16+rsi]
				4043	movdqu xmm2,XMMWORD[rdi]
				4044	movdqu xmm3,XMMWORD[16+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4045	movdqa xmm11,xmm2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4046	movdqu xmm4,XMMWORD[32+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4047	movdqa xmm12,xmm3
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4048	movdqu xmm5,XMMWORD[48+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4049	movdqa xmm13,xmm4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4050	movdqu xmm6,XMMWORD[64+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4051	movdqa xmm14,xmm5
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4052	movdqu xmm7,XMMWORD[80+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4053	movdqa xmm15,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4054	$L$cbc_dec_loop6_enter:
				4055	lea rdi,[96+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4056	movdqa xmm8,xmm7
				4057
				4058	call _aesni_decrypt6
				4059
				4060	pxor xmm2,xmm10
				4061	movdqa xmm10,xmm8
				4062	pxor xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4063	movdqu XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4064	pxor xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4065	movdqu XMMWORD[16+rsi],xmm3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4066	pxor xmm5,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4067	movdqu XMMWORD[32+rsi],xmm4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4068	pxor xmm6,xmm14
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	4069	mov rcx,rbp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4070	movdqu XMMWORD[48+rsi],xmm5
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4071	pxor xmm7,xmm15
				4072	mov eax,r10d
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4073	movdqu XMMWORD[64+rsi],xmm6
				4074	lea rsi,[80+rsi]
				4075	sub rdx,0x60
				4076	ja NEAR $L$cbc_dec_loop6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4077
				4078	movdqa xmm2,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4079	add rdx,0x50
				4080	jle NEAR $L$cbc_dec_clear_tail_collected
				4081	movups XMMWORD[rsi],xmm7
				4082	lea rsi,[16+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4083
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4084	$L$cbc_dec_tail:
				4085	movups xmm2,XMMWORD[rdi]
				4086	sub rdx,0x10
				4087	jbe NEAR $L$cbc_dec_one
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4088
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4089	movups xmm3,XMMWORD[16+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4090	movaps xmm11,xmm2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4091	sub rdx,0x10
				4092	jbe NEAR $L$cbc_dec_two
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4093
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4094	movups xmm4,XMMWORD[32+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4095	movaps xmm12,xmm3
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4096	sub rdx,0x10
				4097	jbe NEAR $L$cbc_dec_three
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4098
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4099	movups xmm5,XMMWORD[48+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4100	movaps xmm13,xmm4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4101	sub rdx,0x10
				4102	jbe NEAR $L$cbc_dec_four
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4103
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4104	movups xmm6,XMMWORD[64+rdi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4105	movaps xmm14,xmm5
				4106	movaps xmm15,xmm6
				4107	xorps xmm7,xmm7
				4108	call _aesni_decrypt6
				4109	pxor xmm2,xmm10
				4110	movaps xmm10,xmm15
				4111	pxor xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4112	movdqu XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4113	pxor xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4114	movdqu XMMWORD[16+rsi],xmm3
				4115	pxor xmm3,xmm3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4116	pxor xmm5,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4117	movdqu XMMWORD[32+rsi],xmm4
				4118	pxor xmm4,xmm4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4119	pxor xmm6,xmm14
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4120	movdqu XMMWORD[48+rsi],xmm5
				4121	pxor xmm5,xmm5
				4122	lea rsi,[64+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4123	movdqa xmm2,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4124	pxor xmm6,xmm6
				4125	pxor xmm7,xmm7
				4126	sub rdx,0x10
				4127	jmp NEAR $L$cbc_dec_tail_collected
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4128
				4129	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4130	$L$cbc_dec_one:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4131	movaps xmm11,xmm2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4132	movups xmm0,XMMWORD[rcx]
				4133	movups xmm1,XMMWORD[16+rcx]
				4134	lea rcx,[32+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4135	xorps xmm2,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4136	$L$oop_dec1_17:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4137	DB 102,15,56,222,209
				4138	dec eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4139	movups xmm1,XMMWORD[rcx]
				4140	lea rcx,[16+rcx]
				4141	jnz NEAR $L$oop_dec1_17
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4142	DB 102,15,56,223,209
				4143	xorps xmm2,xmm10
				4144	movaps xmm10,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4145	jmp NEAR $L$cbc_dec_tail_collected
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4146	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4147	$L$cbc_dec_two:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4148	movaps xmm12,xmm3
				4149	call _aesni_decrypt2
				4150	pxor xmm2,xmm10
				4151	movaps xmm10,xmm12
				4152	pxor xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4153	movdqu XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4154	movdqa xmm2,xmm3
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4155	pxor xmm3,xmm3
				4156	lea rsi,[16+rsi]
				4157	jmp NEAR $L$cbc_dec_tail_collected
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4158	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4159	$L$cbc_dec_three:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4160	movaps xmm13,xmm4
				4161	call _aesni_decrypt3
				4162	pxor xmm2,xmm10
				4163	movaps xmm10,xmm13
				4164	pxor xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4165	movdqu XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4166	pxor xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4167	movdqu XMMWORD[16+rsi],xmm3
				4168	pxor xmm3,xmm3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4169	movdqa xmm2,xmm4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4170	pxor xmm4,xmm4
				4171	lea rsi,[32+rsi]
				4172	jmp NEAR $L$cbc_dec_tail_collected
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4173	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4174	$L$cbc_dec_four:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4175	movaps xmm14,xmm5
				4176	call _aesni_decrypt4
				4177	pxor xmm2,xmm10
				4178	movaps xmm10,xmm14
				4179	pxor xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4180	movdqu XMMWORD[rsi],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4181	pxor xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4182	movdqu XMMWORD[16+rsi],xmm3
				4183	pxor xmm3,xmm3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4184	pxor xmm5,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4185	movdqu XMMWORD[32+rsi],xmm4
				4186	pxor xmm4,xmm4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4187	movdqa xmm2,xmm5
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4188	pxor xmm5,xmm5
				4189	lea rsi,[48+rsi]
				4190	jmp NEAR $L$cbc_dec_tail_collected
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4191
				4192	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4193	$L$cbc_dec_clear_tail_collected:
				4194	pxor xmm3,xmm3
				4195	pxor xmm4,xmm4
				4196	pxor xmm5,xmm5
				4197	$L$cbc_dec_tail_collected:
				4198	movups XMMWORD[r8],xmm10
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4199	and rdx,15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4200	jnz NEAR $L$cbc_dec_tail_partial
				4201	movups XMMWORD[rsi],xmm2
				4202	pxor xmm2,xmm2
				4203	jmp NEAR $L$cbc_dec_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4204	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4205	$L$cbc_dec_tail_partial:
				4206	movaps XMMWORD[rsp],xmm2
				4207	pxor xmm2,xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4208	mov rcx,16
				4209	mov rdi,rsi
				4210	sub rcx,rdx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4211	lea rsi,[rsp]
				4212	DD 0x9066A4F3
				4213	movdqa XMMWORD[rsp],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4214
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4215	$L$cbc_dec_ret:
				4216	xorps xmm0,xmm0
				4217	pxor xmm1,xmm1
				4218	movaps xmm6,XMMWORD[16+rsp]
				4219	movaps XMMWORD[16+rsp],xmm0
				4220	movaps xmm7,XMMWORD[32+rsp]
				4221	movaps XMMWORD[32+rsp],xmm0
				4222	movaps xmm8,XMMWORD[48+rsp]
				4223	movaps XMMWORD[48+rsp],xmm0
				4224	movaps xmm9,XMMWORD[64+rsp]
				4225	movaps XMMWORD[64+rsp],xmm0
				4226	movaps xmm10,XMMWORD[80+rsp]
				4227	movaps XMMWORD[80+rsp],xmm0
				4228	movaps xmm11,XMMWORD[96+rsp]
				4229	movaps XMMWORD[96+rsp],xmm0
				4230	movaps xmm12,XMMWORD[112+rsp]
				4231	movaps XMMWORD[112+rsp],xmm0
				4232	movaps xmm13,XMMWORD[128+rsp]
				4233	movaps XMMWORD[128+rsp],xmm0
				4234	movaps xmm14,XMMWORD[144+rsp]
				4235	movaps XMMWORD[144+rsp],xmm0
				4236	movaps xmm15,XMMWORD[160+rsp]
				4237	movaps XMMWORD[160+rsp],xmm0
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	4238	mov rbp,QWORD[((-8))+r11]
				4239	lea rsp,[r11]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4240	$L$cbc_ret:
				4241	mov rdi,QWORD[8+rsp] ;WIN64 epilogue
				4242	mov rsi,QWORD[16+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4243	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4244	$L$SEH_end_aesni_cbc_encrypt:
				4245	global aesni_set_decrypt_key
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4246
				4247	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4248	aesni_set_decrypt_key:
				4249	DB 0x48,0x83,0xEC,0x08
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4250	call __aesni_set_encrypt_key
				4251	shl edx,4
				4252	test eax,eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4253	jnz NEAR $L$dec_key_ret
				4254	lea rcx,[16+rdx*1+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4255
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4256	movups xmm0,XMMWORD[r8]
				4257	movups xmm1,XMMWORD[rcx]
				4258	movups XMMWORD[rcx],xmm0
				4259	movups XMMWORD[r8],xmm1
				4260	lea r8,[16+r8]
				4261	lea rcx,[((-16))+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4262
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4263	$L$dec_key_inverse:
				4264	movups xmm0,XMMWORD[r8]
				4265	movups xmm1,XMMWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4266	DB 102,15,56,219,192
				4267	DB 102,15,56,219,201
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4268	lea r8,[16+r8]
				4269	lea rcx,[((-16))+rcx]
				4270	movups XMMWORD[16+rcx],xmm0
				4271	movups XMMWORD[(-16)+r8],xmm1
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4272	cmp rcx,r8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4273	ja NEAR $L$dec_key_inverse
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4274
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4275	movups xmm0,XMMWORD[r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4276	DB 102,15,56,219,192
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4277	pxor xmm1,xmm1
				4278	movups XMMWORD[rcx],xmm0
				4279	pxor xmm0,xmm0
				4280	$L$dec_key_ret:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4281	add rsp,8
				4282	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4283	$L$SEH_end_set_decrypt_key:
				4284
				4285	global aesni_set_encrypt_key
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4286
				4287	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4288	aesni_set_encrypt_key:
				4289	__aesni_set_encrypt_key:
				4290	DB 0x48,0x83,0xEC,0x08
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4291	mov rax,-1
				4292	test rcx,rcx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4293	jz NEAR $L$enc_key_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4294	test r8,r8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4295	jz NEAR $L$enc_key_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4296
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4297	movups xmm0,XMMWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4298	xorps xmm4,xmm4
Robert Sloan	2424d84	2017-05-01 07:46:28 -0700	[diff] [blame]	4299	lea r10,[OPENSSL_ia32cap_P]
Robert Sloan	572a4e2	2017-04-17 10:52:19 -0700	[diff] [blame]	4300	mov r10d,DWORD[4+r10]
				4301	and r10d,268437504
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4302	lea rax,[16+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4303	cmp edx,256
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4304	je NEAR $L$14rounds
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4305	cmp edx,192
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4306	je NEAR $L$12rounds
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4307	cmp edx,128
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4308	jne NEAR $L$bad_keybits
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4309
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4310	$L$10rounds:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4311	mov edx,9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4312	cmp r10d,268435456
				4313	je NEAR $L$10rounds_alt
				4314
				4315	movups XMMWORD[r8],xmm0
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4316	DB 102,15,58,223,200,1
				4317	call $L$key_expansion_128_cold
				4318	DB 102,15,58,223,200,2
				4319	call $L$key_expansion_128
				4320	DB 102,15,58,223,200,4
				4321	call $L$key_expansion_128
				4322	DB 102,15,58,223,200,8
				4323	call $L$key_expansion_128
				4324	DB 102,15,58,223,200,16
				4325	call $L$key_expansion_128
				4326	DB 102,15,58,223,200,32
				4327	call $L$key_expansion_128
				4328	DB 102,15,58,223,200,64
				4329	call $L$key_expansion_128
				4330	DB 102,15,58,223,200,128
				4331	call $L$key_expansion_128
				4332	DB 102,15,58,223,200,27
				4333	call $L$key_expansion_128
				4334	DB 102,15,58,223,200,54
				4335	call $L$key_expansion_128
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4336	movups XMMWORD[rax],xmm0
				4337	mov DWORD[80+rax],edx
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4338	xor eax,eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4339	jmp NEAR $L$enc_key_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4340
				4341	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4342	$L$10rounds_alt:
				4343	movdqa xmm5,XMMWORD[$L$key_rotate]
				4344	mov r10d,8
				4345	movdqa xmm4,XMMWORD[$L$key_rcon1]
				4346	movdqa xmm2,xmm0
				4347	movdqu XMMWORD[r8],xmm0
				4348	jmp NEAR $L$oop_key128
				4349
				4350	ALIGN 16
				4351	$L$oop_key128:
				4352	DB 102,15,56,0,197
				4353	DB 102,15,56,221,196
				4354	pslld xmm4,1
				4355	lea rax,[16+rax]
				4356
				4357	movdqa xmm3,xmm2
				4358	pslldq xmm2,4
				4359	pxor xmm3,xmm2
				4360	pslldq xmm2,4
				4361	pxor xmm3,xmm2
				4362	pslldq xmm2,4
				4363	pxor xmm2,xmm3
				4364
				4365	pxor xmm0,xmm2
				4366	movdqu XMMWORD[(-16)+rax],xmm0
				4367	movdqa xmm2,xmm0
				4368
				4369	dec r10d
				4370	jnz NEAR $L$oop_key128
				4371
				4372	movdqa xmm4,XMMWORD[$L$key_rcon1b]
				4373
				4374	DB 102,15,56,0,197
				4375	DB 102,15,56,221,196
				4376	pslld xmm4,1
				4377
				4378	movdqa xmm3,xmm2
				4379	pslldq xmm2,4
				4380	pxor xmm3,xmm2
				4381	pslldq xmm2,4
				4382	pxor xmm3,xmm2
				4383	pslldq xmm2,4
				4384	pxor xmm2,xmm3
				4385
				4386	pxor xmm0,xmm2
				4387	movdqu XMMWORD[rax],xmm0
				4388
				4389	movdqa xmm2,xmm0
				4390	DB 102,15,56,0,197
				4391	DB 102,15,56,221,196
				4392
				4393	movdqa xmm3,xmm2
				4394	pslldq xmm2,4
				4395	pxor xmm3,xmm2
				4396	pslldq xmm2,4
				4397	pxor xmm3,xmm2
				4398	pslldq xmm2,4
				4399	pxor xmm2,xmm3
				4400
				4401	pxor xmm0,xmm2
				4402	movdqu XMMWORD[16+rax],xmm0
				4403
				4404	mov DWORD[96+rax],edx
				4405	xor eax,eax
				4406	jmp NEAR $L$enc_key_ret
				4407
				4408	ALIGN 16
				4409	$L$12rounds:
				4410	movq xmm2,QWORD[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4411	mov edx,11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4412	cmp r10d,268435456
				4413	je NEAR $L$12rounds_alt
				4414
				4415	movups XMMWORD[r8],xmm0
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4416	DB 102,15,58,223,202,1
				4417	call $L$key_expansion_192a_cold
				4418	DB 102,15,58,223,202,2
				4419	call $L$key_expansion_192b
				4420	DB 102,15,58,223,202,4
				4421	call $L$key_expansion_192a
				4422	DB 102,15,58,223,202,8
				4423	call $L$key_expansion_192b
				4424	DB 102,15,58,223,202,16
				4425	call $L$key_expansion_192a
				4426	DB 102,15,58,223,202,32
				4427	call $L$key_expansion_192b
				4428	DB 102,15,58,223,202,64
				4429	call $L$key_expansion_192a
				4430	DB 102,15,58,223,202,128
				4431	call $L$key_expansion_192b
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4432	movups XMMWORD[rax],xmm0
				4433	mov DWORD[48+rax],edx
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4434	xor rax,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4435	jmp NEAR $L$enc_key_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4436
				4437	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4438	$L$12rounds_alt:
				4439	movdqa xmm5,XMMWORD[$L$key_rotate192]
				4440	movdqa xmm4,XMMWORD[$L$key_rcon1]
				4441	mov r10d,8
				4442	movdqu XMMWORD[r8],xmm0
				4443	jmp NEAR $L$oop_key192
				4444
				4445	ALIGN 16
				4446	$L$oop_key192:
				4447	movq QWORD[rax],xmm2
				4448	movdqa xmm1,xmm2
				4449	DB 102,15,56,0,213
				4450	DB 102,15,56,221,212
				4451	pslld xmm4,1
				4452	lea rax,[24+rax]
				4453
				4454	movdqa xmm3,xmm0
				4455	pslldq xmm0,4
				4456	pxor xmm3,xmm0
				4457	pslldq xmm0,4
				4458	pxor xmm3,xmm0
				4459	pslldq xmm0,4
				4460	pxor xmm0,xmm3
				4461
				4462	pshufd xmm3,xmm0,0xff
				4463	pxor xmm3,xmm1
				4464	pslldq xmm1,4
				4465	pxor xmm3,xmm1
				4466
				4467	pxor xmm0,xmm2
				4468	pxor xmm2,xmm3
				4469	movdqu XMMWORD[(-16)+rax],xmm0
				4470
				4471	dec r10d
				4472	jnz NEAR $L$oop_key192
				4473
				4474	mov DWORD[32+rax],edx
				4475	xor eax,eax
				4476	jmp NEAR $L$enc_key_ret
				4477
				4478	ALIGN 16
				4479	$L$14rounds:
				4480	movups xmm2,XMMWORD[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4481	mov edx,13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4482	lea rax,[16+rax]
				4483	cmp r10d,268435456
				4484	je NEAR $L$14rounds_alt
				4485
				4486	movups XMMWORD[r8],xmm0
				4487	movups XMMWORD[16+r8],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4488	DB 102,15,58,223,202,1
				4489	call $L$key_expansion_256a_cold
				4490	DB 102,15,58,223,200,1
				4491	call $L$key_expansion_256b
				4492	DB 102,15,58,223,202,2
				4493	call $L$key_expansion_256a
				4494	DB 102,15,58,223,200,2
				4495	call $L$key_expansion_256b
				4496	DB 102,15,58,223,202,4
				4497	call $L$key_expansion_256a
				4498	DB 102,15,58,223,200,4
				4499	call $L$key_expansion_256b
				4500	DB 102,15,58,223,202,8
				4501	call $L$key_expansion_256a
				4502	DB 102,15,58,223,200,8
				4503	call $L$key_expansion_256b
				4504	DB 102,15,58,223,202,16
				4505	call $L$key_expansion_256a
				4506	DB 102,15,58,223,200,16
				4507	call $L$key_expansion_256b
				4508	DB 102,15,58,223,202,32
				4509	call $L$key_expansion_256a
				4510	DB 102,15,58,223,200,32
				4511	call $L$key_expansion_256b
				4512	DB 102,15,58,223,202,64
				4513	call $L$key_expansion_256a
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4514	movups XMMWORD[rax],xmm0
				4515	mov DWORD[16+rax],edx
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4516	xor rax,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4517	jmp NEAR $L$enc_key_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4518
				4519	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4520	$L$14rounds_alt:
				4521	movdqa xmm5,XMMWORD[$L$key_rotate]
				4522	movdqa xmm4,XMMWORD[$L$key_rcon1]
				4523	mov r10d,7
				4524	movdqu XMMWORD[r8],xmm0
				4525	movdqa xmm1,xmm2
				4526	movdqu XMMWORD[16+r8],xmm2
				4527	jmp NEAR $L$oop_key256
				4528
				4529	ALIGN 16
				4530	$L$oop_key256:
				4531	DB 102,15,56,0,213
				4532	DB 102,15,56,221,212
				4533
				4534	movdqa xmm3,xmm0
				4535	pslldq xmm0,4
				4536	pxor xmm3,xmm0
				4537	pslldq xmm0,4
				4538	pxor xmm3,xmm0
				4539	pslldq xmm0,4
				4540	pxor xmm0,xmm3
				4541	pslld xmm4,1
				4542
				4543	pxor xmm0,xmm2
				4544	movdqu XMMWORD[rax],xmm0
				4545
				4546	dec r10d
				4547	jz NEAR $L$done_key256
				4548
				4549	pshufd xmm2,xmm0,0xff
				4550	pxor xmm3,xmm3
				4551	DB 102,15,56,221,211
				4552
				4553	movdqa xmm3,xmm1
				4554	pslldq xmm1,4
				4555	pxor xmm3,xmm1
				4556	pslldq xmm1,4
				4557	pxor xmm3,xmm1
				4558	pslldq xmm1,4
				4559	pxor xmm1,xmm3
				4560
				4561	pxor xmm2,xmm1
				4562	movdqu XMMWORD[16+rax],xmm2
				4563	lea rax,[32+rax]
				4564	movdqa xmm1,xmm2
				4565
				4566	jmp NEAR $L$oop_key256
				4567
				4568	$L$done_key256:
				4569	mov DWORD[16+rax],edx
				4570	xor eax,eax
				4571	jmp NEAR $L$enc_key_ret
				4572
				4573	ALIGN 16
				4574	$L$bad_keybits:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4575	mov rax,-2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4576	$L$enc_key_ret:
				4577	pxor xmm0,xmm0
				4578	pxor xmm1,xmm1
				4579	pxor xmm2,xmm2
				4580	pxor xmm3,xmm3
				4581	pxor xmm4,xmm4
				4582	pxor xmm5,xmm5
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4583	add rsp,8
				4584	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4585	$L$SEH_end_set_encrypt_key:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4586
				4587	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4588	$L$key_expansion_128:
				4589	movups XMMWORD[rax],xmm0
				4590	lea rax,[16+rax]
				4591	$L$key_expansion_128_cold:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4592	shufps xmm4,xmm0,16
				4593	xorps xmm0,xmm4
				4594	shufps xmm4,xmm0,140
				4595	xorps xmm0,xmm4
				4596	shufps xmm1,xmm1,255
				4597	xorps xmm0,xmm1
				4598	DB 0F3h,0C3h ;repret
				4599
				4600	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4601	$L$key_expansion_192a:
				4602	movups XMMWORD[rax],xmm0
				4603	lea rax,[16+rax]
				4604	$L$key_expansion_192a_cold:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4605	movaps xmm5,xmm2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4606	$L$key_expansion_192b_warm:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4607	shufps xmm4,xmm0,16
				4608	movdqa xmm3,xmm2
				4609	xorps xmm0,xmm4
				4610	shufps xmm4,xmm0,140
				4611	pslldq xmm3,4
				4612	xorps xmm0,xmm4
				4613	pshufd xmm1,xmm1,85
				4614	pxor xmm2,xmm3
				4615	pxor xmm0,xmm1
				4616	pshufd xmm3,xmm0,255
				4617	pxor xmm2,xmm3
				4618	DB 0F3h,0C3h ;repret
				4619
				4620	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4621	$L$key_expansion_192b:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4622	movaps xmm3,xmm0
				4623	shufps xmm5,xmm0,68
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4624	movups XMMWORD[rax],xmm5
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4625	shufps xmm3,xmm2,78
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4626	movups XMMWORD[16+rax],xmm3
				4627	lea rax,[32+rax]
				4628	jmp NEAR $L$key_expansion_192b_warm
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4629
				4630	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4631	$L$key_expansion_256a:
				4632	movups XMMWORD[rax],xmm2
				4633	lea rax,[16+rax]
				4634	$L$key_expansion_256a_cold:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4635	shufps xmm4,xmm0,16
				4636	xorps xmm0,xmm4
				4637	shufps xmm4,xmm0,140
				4638	xorps xmm0,xmm4
				4639	shufps xmm1,xmm1,255
				4640	xorps xmm0,xmm1
				4641	DB 0F3h,0C3h ;repret
				4642
				4643	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4644	$L$key_expansion_256b:
				4645	movups XMMWORD[rax],xmm0
				4646	lea rax,[16+rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4647
				4648	shufps xmm4,xmm2,16
				4649	xorps xmm2,xmm4
				4650	shufps xmm4,xmm2,140
				4651	xorps xmm2,xmm4
				4652	shufps xmm1,xmm1,170
				4653	xorps xmm2,xmm1
				4654	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4655
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4656
				4657	ALIGN 64
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4658	$L$bswap_mask:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4659	DB 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4660	$L$increment32:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4661	DD 6,6,6,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4662	$L$increment64:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4663	DD 1,0,0,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4664	$L$xts_magic:
				4665	DD 0x87,0,1,0
				4666	$L$increment1:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4667	DB 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4668	$L$key_rotate:
				4669	DD 0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d
				4670	$L$key_rotate192:
				4671	DD 0x04070605,0x04070605,0x04070605,0x04070605
				4672	$L$key_rcon1:
				4673	DD 1,1,1,1
				4674	$L$key_rcon1b:
				4675	DD 0x1b,0x1b,0x1b,0x1b
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4676
				4677	DB 65,69,83,32,102,111,114,32,73,110,116,101,108,32,65,69
				4678	DB 83,45,78,73,44,32,67,82,89,80,84,79,71,65,77,83
				4679	DB 32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115
				4680	DB 115,108,46,111,114,103,62,0
				4681	ALIGN 64
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4682	EXTERN __imp_RtlVirtualUnwind
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4683
				4684	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4685	ecb_ccm64_se_handler:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4686	push rsi
				4687	push rdi
				4688	push rbx
				4689	push rbp
				4690	push r12
				4691	push r13
				4692	push r14
				4693	push r15
				4694	pushfq
				4695	sub rsp,64
				4696
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4697	mov rax,QWORD[120+r8]
				4698	mov rbx,QWORD[248+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4699
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4700	mov rsi,QWORD[8+r9]
				4701	mov r11,QWORD[56+r9]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4702
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4703	mov r10d,DWORD[r11]
				4704	lea r10,[r10*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4705	cmp rbx,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4706	jb NEAR $L$common_seh_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4707
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4708	mov rax,QWORD[152+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4709
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4710	mov r10d,DWORD[4+r11]
				4711	lea r10,[r10*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4712	cmp rbx,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4713	jae NEAR $L$common_seh_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4714
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4715	lea rsi,[rax]
				4716	lea rdi,[512+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4717	mov ecx,8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4718	DD 0xa548f3fc
				4719	lea rax,[88+rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4720
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4721	jmp NEAR $L$common_seh_tail
				4722
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4723
				4724
				4725	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4726	ctr_xts_se_handler:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4727	push rsi
				4728	push rdi
				4729	push rbx
				4730	push rbp
				4731	push r12
				4732	push r13
				4733	push r14
				4734	push r15
				4735	pushfq
				4736	sub rsp,64
				4737
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4738	mov rax,QWORD[120+r8]
				4739	mov rbx,QWORD[248+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4740
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4741	mov rsi,QWORD[8+r9]
				4742	mov r11,QWORD[56+r9]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4743
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4744	mov r10d,DWORD[r11]
				4745	lea r10,[r10*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4746	cmp rbx,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4747	jb NEAR $L$common_seh_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4748
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4749	mov rax,QWORD[152+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4750
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4751	mov r10d,DWORD[4+r11]
				4752	lea r10,[r10*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4753	cmp rbx,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4754	jae NEAR $L$common_seh_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4755
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	4756	mov rax,QWORD[208+r8]
				4757
				4758	lea rsi,[((-168))+rax]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4759	lea rdi,[512+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4760	mov ecx,20
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4761	DD 0xa548f3fc
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4762
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	4763	mov rbp,QWORD[((-8))+rax]
				4764	mov QWORD[160+r8],rbp
				4765	jmp NEAR $L$common_seh_tail
				4766
				4767
				4768
				4769	ALIGN 16
				4770	ocb_se_handler:
				4771	push rsi
				4772	push rdi
				4773	push rbx
				4774	push rbp
				4775	push r12
				4776	push r13
				4777	push r14
				4778	push r15
				4779	pushfq
				4780	sub rsp,64
				4781
				4782	mov rax,QWORD[120+r8]
				4783	mov rbx,QWORD[248+r8]
				4784
				4785	mov rsi,QWORD[8+r9]
				4786	mov r11,QWORD[56+r9]
				4787
				4788	mov r10d,DWORD[r11]
				4789	lea r10,[r10*1+rsi]
				4790	cmp rbx,r10
				4791	jb NEAR $L$common_seh_tail
				4792
				4793	mov r10d,DWORD[4+r11]
				4794	lea r10,[r10*1+rsi]
				4795	cmp rbx,r10
				4796	jae NEAR $L$common_seh_tail
				4797
				4798	mov r10d,DWORD[8+r11]
				4799	lea r10,[r10*1+rsi]
				4800	cmp rbx,r10
				4801	jae NEAR $L$ocb_no_xmm
				4802
				4803	mov rax,QWORD[152+r8]
				4804
				4805	lea rsi,[rax]
				4806	lea rdi,[512+r8]
				4807	mov ecx,20
				4808	DD 0xa548f3fc
				4809	lea rax,[((160+40))+rax]
				4810
				4811	$L$ocb_no_xmm:
				4812	mov rbx,QWORD[((-8))+rax]
				4813	mov rbp,QWORD[((-16))+rax]
				4814	mov r12,QWORD[((-24))+rax]
				4815	mov r13,QWORD[((-32))+rax]
				4816	mov r14,QWORD[((-40))+rax]
				4817
				4818	mov QWORD[144+r8],rbx
				4819	mov QWORD[160+r8],rbp
				4820	mov QWORD[216+r8],r12
				4821	mov QWORD[224+r8],r13
				4822	mov QWORD[232+r8],r14
				4823
				4824	jmp NEAR $L$common_seh_tail
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4825
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4826
				4827	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4828	cbc_se_handler:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4829	push rsi
				4830	push rdi
				4831	push rbx
				4832	push rbp
				4833	push r12
				4834	push r13
				4835	push r14
				4836	push r15
				4837	pushfq
				4838	sub rsp,64
				4839
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4840	mov rax,QWORD[152+r8]
				4841	mov rbx,QWORD[248+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4842
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4843	lea r10,[$L$cbc_decrypt_bulk]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4844	cmp rbx,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4845	jb NEAR $L$common_seh_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4846
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	4847	mov rax,QWORD[120+r8]
				4848
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4849	lea r10,[$L$cbc_decrypt_body]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4850	cmp rbx,r10
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	4851	jb NEAR $L$common_seh_tail
				4852
				4853	mov rax,QWORD[152+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4854
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4855	lea r10,[$L$cbc_ret]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4856	cmp rbx,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4857	jae NEAR $L$common_seh_tail
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4858
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4859	lea rsi,[16+rax]
				4860	lea rdi,[512+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4861	mov ecx,20
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4862	DD 0xa548f3fc
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4863
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	4864	mov rax,QWORD[208+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4865
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	4866	mov rbp,QWORD[((-8))+rax]
				4867	mov QWORD[160+r8],rbp
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4868
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4869	$L$common_seh_tail:
				4870	mov rdi,QWORD[8+rax]
				4871	mov rsi,QWORD[16+rax]
				4872	mov QWORD[152+r8],rax
				4873	mov QWORD[168+r8],rsi
				4874	mov QWORD[176+r8],rdi
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4875
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4876	mov rdi,QWORD[40+r9]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4877	mov rsi,r8
				4878	mov ecx,154
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4879	DD 0xa548f3fc
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4880
				4881	mov rsi,r9
				4882	xor rcx,rcx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4883	mov rdx,QWORD[8+rsi]
				4884	mov r8,QWORD[rsi]
				4885	mov r9,QWORD[16+rsi]
				4886	mov r10,QWORD[40+rsi]
				4887	lea r11,[56+rsi]
				4888	lea r12,[24+rsi]
				4889	mov QWORD[32+rsp],r10
				4890	mov QWORD[40+rsp],r11
				4891	mov QWORD[48+rsp],r12
				4892	mov QWORD[56+rsp],rcx
				4893	call QWORD[__imp_RtlVirtualUnwind]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4894
				4895	mov eax,1
				4896	add rsp,64
				4897	popfq
				4898	pop r15
				4899	pop r14
				4900	pop r13
				4901	pop r12
				4902	pop rbp
				4903	pop rbx
				4904	pop rdi
				4905	pop rsi
				4906	DB 0F3h,0C3h ;repret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4907
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4908
				4909	section .pdata rdata align=4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4910	ALIGN 4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4911	DD $L$SEH_begin_aesni_ecb_encrypt wrt ..imagebase
				4912	DD $L$SEH_end_aesni_ecb_encrypt wrt ..imagebase
				4913	DD $L$SEH_info_ecb wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4914
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4915	DD $L$SEH_begin_aesni_ccm64_encrypt_blocks wrt ..imagebase
				4916	DD $L$SEH_end_aesni_ccm64_encrypt_blocks wrt ..imagebase
				4917	DD $L$SEH_info_ccm64_enc wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4918
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4919	DD $L$SEH_begin_aesni_ccm64_decrypt_blocks wrt ..imagebase
				4920	DD $L$SEH_end_aesni_ccm64_decrypt_blocks wrt ..imagebase
				4921	DD $L$SEH_info_ccm64_dec wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4922
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4923	DD $L$SEH_begin_aesni_ctr32_encrypt_blocks wrt ..imagebase
				4924	DD $L$SEH_end_aesni_ctr32_encrypt_blocks wrt ..imagebase
				4925	DD $L$SEH_info_ctr32 wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4926
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4927	DD $L$SEH_begin_aesni_xts_encrypt wrt ..imagebase
				4928	DD $L$SEH_end_aesni_xts_encrypt wrt ..imagebase
				4929	DD $L$SEH_info_xts_enc wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4930
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4931	DD $L$SEH_begin_aesni_xts_decrypt wrt ..imagebase
				4932	DD $L$SEH_end_aesni_xts_decrypt wrt ..imagebase
				4933	DD $L$SEH_info_xts_dec wrt ..imagebase
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	4934
				4935	DD $L$SEH_begin_aesni_ocb_encrypt wrt ..imagebase
				4936	DD $L$SEH_end_aesni_ocb_encrypt wrt ..imagebase
				4937	DD $L$SEH_info_ocb_enc wrt ..imagebase
				4938
				4939	DD $L$SEH_begin_aesni_ocb_decrypt wrt ..imagebase
				4940	DD $L$SEH_end_aesni_ocb_decrypt wrt ..imagebase
				4941	DD $L$SEH_info_ocb_dec wrt ..imagebase
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4942	DD $L$SEH_begin_aesni_cbc_encrypt wrt ..imagebase
				4943	DD $L$SEH_end_aesni_cbc_encrypt wrt ..imagebase
				4944	DD $L$SEH_info_cbc wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4945
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4946	DD aesni_set_decrypt_key wrt ..imagebase
				4947	DD $L$SEH_end_set_decrypt_key wrt ..imagebase
				4948	DD $L$SEH_info_key wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4949
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4950	DD aesni_set_encrypt_key wrt ..imagebase
				4951	DD $L$SEH_end_set_encrypt_key wrt ..imagebase
				4952	DD $L$SEH_info_key wrt ..imagebase
				4953	section .xdata rdata align=8
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4954	ALIGN 8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4955	$L$SEH_info_ecb:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4956	DB 9,0,0,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4957	DD ecb_ccm64_se_handler wrt ..imagebase
				4958	DD $L$ecb_enc_body wrt ..imagebase,$L$ecb_enc_ret wrt ..imagebase
				4959	$L$SEH_info_ccm64_enc:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4960	DB 9,0,0,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4961	DD ecb_ccm64_se_handler wrt ..imagebase
				4962	DD $L$ccm64_enc_body wrt ..imagebase,$L$ccm64_enc_ret wrt ..imagebase
				4963	$L$SEH_info_ccm64_dec:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4964	DB 9,0,0,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4965	DD ecb_ccm64_se_handler wrt ..imagebase
				4966	DD $L$ccm64_dec_body wrt ..imagebase,$L$ccm64_dec_ret wrt ..imagebase
				4967	$L$SEH_info_ctr32:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4968	DB 9,0,0,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4969	DD ctr_xts_se_handler wrt ..imagebase
				4970	DD $L$ctr32_body wrt ..imagebase,$L$ctr32_epilogue wrt ..imagebase
				4971	$L$SEH_info_xts_enc:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4972	DB 9,0,0,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4973	DD ctr_xts_se_handler wrt ..imagebase
				4974	DD $L$xts_enc_body wrt ..imagebase,$L$xts_enc_epilogue wrt ..imagebase
				4975	$L$SEH_info_xts_dec:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4976	DB 9,0,0,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4977	DD ctr_xts_se_handler wrt ..imagebase
				4978	DD $L$xts_dec_body wrt ..imagebase,$L$xts_dec_epilogue wrt ..imagebase
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	4979	$L$SEH_info_ocb_enc:
				4980	DB 9,0,0,0
				4981	DD ocb_se_handler wrt ..imagebase
				4982	DD $L$ocb_enc_body wrt ..imagebase,$L$ocb_enc_epilogue wrt ..imagebase
				4983	DD $L$ocb_enc_pop wrt ..imagebase
				4984	DD 0
				4985	$L$SEH_info_ocb_dec:
				4986	DB 9,0,0,0
				4987	DD ocb_se_handler wrt ..imagebase
				4988	DD $L$ocb_dec_body wrt ..imagebase,$L$ocb_dec_epilogue wrt ..imagebase
				4989	DD $L$ocb_dec_pop wrt ..imagebase
				4990	DD 0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4991	$L$SEH_info_cbc:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	4992	DB 9,0,0,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	4993	DD cbc_se_handler wrt ..imagebase
				4994	$L$SEH_info_key:
				4995	DB 0x01,0x04,0x01,0x00
				4996	DB 0x04,0x02,0x00,0x00