Blame - win-x86_64/crypto/fipsmodule/bsaes-x86_64.asm - platform/external/boringssl

blob: 9ea8253d7c70166655acfa819c7e3bf2b55ec9f5 [file] [log] [blame]

Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1	default rel
				2	%define XMMWORD
				3	%define YMMWORD
				4	%define ZMMWORD
				5	section .text code align=64
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	7
				8	EXTERN asm_AES_encrypt
				9	EXTERN asm_AES_decrypt
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	10
				11
				12	ALIGN 64
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	13	_bsaes_encrypt8:
				14	lea r11,[$L$BS0]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	16	movdqa xmm8,XMMWORD[rax]
				17	lea rax,[16+rax]
				18	movdqa xmm7,XMMWORD[80+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	19	pxor xmm15,xmm8
				20	pxor xmm0,xmm8
				21	pxor xmm1,xmm8
				22	pxor xmm2,xmm8
				23	DB 102,68,15,56,0,255
				24	DB 102,15,56,0,199
				25	pxor xmm3,xmm8
				26	pxor xmm4,xmm8
				27	DB 102,15,56,0,207
				28	DB 102,15,56,0,215
				29	pxor xmm5,xmm8
				30	pxor xmm6,xmm8
				31	DB 102,15,56,0,223
				32	DB 102,15,56,0,231
				33	DB 102,15,56,0,239
				34	DB 102,15,56,0,247
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	35	_bsaes_encrypt8_bitslice:
				36	movdqa xmm7,XMMWORD[r11]
				37	movdqa xmm8,XMMWORD[16+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	38	movdqa xmm9,xmm5
				39	psrlq xmm5,1
				40	movdqa xmm10,xmm3
				41	psrlq xmm3,1
				42	pxor xmm5,xmm6
				43	pxor xmm3,xmm4
				44	pand xmm5,xmm7
				45	pand xmm3,xmm7
				46	pxor xmm6,xmm5
				47	psllq xmm5,1
				48	pxor xmm4,xmm3
				49	psllq xmm3,1
				50	pxor xmm5,xmm9
				51	pxor xmm3,xmm10
				52	movdqa xmm9,xmm1
				53	psrlq xmm1,1
				54	movdqa xmm10,xmm15
				55	psrlq xmm15,1
				56	pxor xmm1,xmm2
				57	pxor xmm15,xmm0
				58	pand xmm1,xmm7
				59	pand xmm15,xmm7
				60	pxor xmm2,xmm1
				61	psllq xmm1,1
				62	pxor xmm0,xmm15
				63	psllq xmm15,1
				64	pxor xmm1,xmm9
				65	pxor xmm15,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	66	movdqa xmm7,XMMWORD[32+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	67	movdqa xmm9,xmm4
				68	psrlq xmm4,2
				69	movdqa xmm10,xmm3
				70	psrlq xmm3,2
				71	pxor xmm4,xmm6
				72	pxor xmm3,xmm5
				73	pand xmm4,xmm8
				74	pand xmm3,xmm8
				75	pxor xmm6,xmm4
				76	psllq xmm4,2
				77	pxor xmm5,xmm3
				78	psllq xmm3,2
				79	pxor xmm4,xmm9
				80	pxor xmm3,xmm10
				81	movdqa xmm9,xmm0
				82	psrlq xmm0,2
				83	movdqa xmm10,xmm15
				84	psrlq xmm15,2
				85	pxor xmm0,xmm2
				86	pxor xmm15,xmm1
				87	pand xmm0,xmm8
				88	pand xmm15,xmm8
				89	pxor xmm2,xmm0
				90	psllq xmm0,2
				91	pxor xmm1,xmm15
				92	psllq xmm15,2
				93	pxor xmm0,xmm9
				94	pxor xmm15,xmm10
				95	movdqa xmm9,xmm2
				96	psrlq xmm2,4
				97	movdqa xmm10,xmm1
				98	psrlq xmm1,4
				99	pxor xmm2,xmm6
				100	pxor xmm1,xmm5
				101	pand xmm2,xmm7
				102	pand xmm1,xmm7
				103	pxor xmm6,xmm2
				104	psllq xmm2,4
				105	pxor xmm5,xmm1
				106	psllq xmm1,4
				107	pxor xmm2,xmm9
				108	pxor xmm1,xmm10
				109	movdqa xmm9,xmm0
				110	psrlq xmm0,4
				111	movdqa xmm10,xmm15
				112	psrlq xmm15,4
				113	pxor xmm0,xmm4
				114	pxor xmm15,xmm3
				115	pand xmm0,xmm7
				116	pand xmm15,xmm7
				117	pxor xmm4,xmm0
				118	psllq xmm0,4
				119	pxor xmm3,xmm15
				120	psllq xmm15,4
				121	pxor xmm0,xmm9
				122	pxor xmm15,xmm10
				123	dec r10d
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	124	jmp NEAR $L$enc_sbox
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	125	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	126	$L$enc_loop:
				127	pxor xmm15,XMMWORD[rax]
				128	pxor xmm0,XMMWORD[16+rax]
				129	pxor xmm1,XMMWORD[32+rax]
				130	pxor xmm2,XMMWORD[48+rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	131	DB 102,68,15,56,0,255
				132	DB 102,15,56,0,199
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	133	pxor xmm3,XMMWORD[64+rax]
				134	pxor xmm4,XMMWORD[80+rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	135	DB 102,15,56,0,207
				136	DB 102,15,56,0,215
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	137	pxor xmm5,XMMWORD[96+rax]
				138	pxor xmm6,XMMWORD[112+rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	139	DB 102,15,56,0,223
				140	DB 102,15,56,0,231
				141	DB 102,15,56,0,239
				142	DB 102,15,56,0,247
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	143	lea rax,[128+rax]
				144	$L$enc_sbox:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	145	pxor xmm4,xmm5
				146	pxor xmm1,xmm0
				147	pxor xmm2,xmm15
				148	pxor xmm5,xmm1
				149	pxor xmm4,xmm15
				150
				151	pxor xmm5,xmm2
				152	pxor xmm2,xmm6
				153	pxor xmm6,xmm4
				154	pxor xmm2,xmm3
				155	pxor xmm3,xmm4
				156	pxor xmm2,xmm0
				157
				158	pxor xmm1,xmm6
				159	pxor xmm0,xmm4
				160	movdqa xmm10,xmm6
				161	movdqa xmm9,xmm0
				162	movdqa xmm8,xmm4
				163	movdqa xmm12,xmm1
				164	movdqa xmm11,xmm5
				165
				166	pxor xmm10,xmm3
				167	pxor xmm9,xmm1
				168	pxor xmm8,xmm2
				169	movdqa xmm13,xmm10
				170	pxor xmm12,xmm3
				171	movdqa xmm7,xmm9
				172	pxor xmm11,xmm15
				173	movdqa xmm14,xmm10
				174
				175	por xmm9,xmm8
				176	por xmm10,xmm11
				177	pxor xmm14,xmm7
				178	pand xmm13,xmm11
				179	pxor xmm11,xmm8
				180	pand xmm7,xmm8
				181	pand xmm14,xmm11
				182	movdqa xmm11,xmm2
				183	pxor xmm11,xmm15
				184	pand xmm12,xmm11
				185	pxor xmm10,xmm12
				186	pxor xmm9,xmm12
				187	movdqa xmm12,xmm6
				188	movdqa xmm11,xmm4
				189	pxor xmm12,xmm0
				190	pxor xmm11,xmm5
				191	movdqa xmm8,xmm12
				192	pand xmm12,xmm11
				193	por xmm8,xmm11
				194	pxor xmm7,xmm12
				195	pxor xmm10,xmm14
				196	pxor xmm9,xmm13
				197	pxor xmm8,xmm14
				198	movdqa xmm11,xmm1
				199	pxor xmm7,xmm13
				200	movdqa xmm12,xmm3
				201	pxor xmm8,xmm13
				202	movdqa xmm13,xmm0
				203	pand xmm11,xmm2
				204	movdqa xmm14,xmm6
				205	pand xmm12,xmm15
				206	pand xmm13,xmm4
				207	por xmm14,xmm5
				208	pxor xmm10,xmm11
				209	pxor xmm9,xmm12
				210	pxor xmm8,xmm13
				211	pxor xmm7,xmm14
				212
				213
				214
				215
				216
				217	movdqa xmm11,xmm10
				218	pand xmm10,xmm8
				219	pxor xmm11,xmm9
				220
				221	movdqa xmm13,xmm7
				222	movdqa xmm14,xmm11
				223	pxor xmm13,xmm10
				224	pand xmm14,xmm13
				225
				226	movdqa xmm12,xmm8
				227	pxor xmm14,xmm9
				228	pxor xmm12,xmm7
				229
				230	pxor xmm10,xmm9
				231
				232	pand xmm12,xmm10
				233
				234	movdqa xmm9,xmm13
				235	pxor xmm12,xmm7
				236
				237	pxor xmm9,xmm12
				238	pxor xmm8,xmm12
				239
				240	pand xmm9,xmm7
				241
				242	pxor xmm13,xmm9
				243	pxor xmm8,xmm9
				244
				245	pand xmm13,xmm14
				246
				247	pxor xmm13,xmm11
				248	movdqa xmm11,xmm5
				249	movdqa xmm7,xmm4
				250	movdqa xmm9,xmm14
				251	pxor xmm9,xmm13
				252	pand xmm9,xmm5
				253	pxor xmm5,xmm4
				254	pand xmm4,xmm14
				255	pand xmm5,xmm13
				256	pxor xmm5,xmm4
				257	pxor xmm4,xmm9
				258	pxor xmm11,xmm15
				259	pxor xmm7,xmm2
				260	pxor xmm14,xmm12
				261	pxor xmm13,xmm8
				262	movdqa xmm10,xmm14
				263	movdqa xmm9,xmm12
				264	pxor xmm10,xmm13
				265	pxor xmm9,xmm8
				266	pand xmm10,xmm11
				267	pand xmm9,xmm15
				268	pxor xmm11,xmm7
				269	pxor xmm15,xmm2
				270	pand xmm7,xmm14
				271	pand xmm2,xmm12
				272	pand xmm11,xmm13
				273	pand xmm15,xmm8
				274	pxor xmm7,xmm11
				275	pxor xmm15,xmm2
				276	pxor xmm11,xmm10
				277	pxor xmm2,xmm9
				278	pxor xmm5,xmm11
				279	pxor xmm15,xmm11
				280	pxor xmm4,xmm7
				281	pxor xmm2,xmm7
				282
				283	movdqa xmm11,xmm6
				284	movdqa xmm7,xmm0
				285	pxor xmm11,xmm3
				286	pxor xmm7,xmm1
				287	movdqa xmm10,xmm14
				288	movdqa xmm9,xmm12
				289	pxor xmm10,xmm13
				290	pxor xmm9,xmm8
				291	pand xmm10,xmm11
				292	pand xmm9,xmm3
				293	pxor xmm11,xmm7
				294	pxor xmm3,xmm1
				295	pand xmm7,xmm14
				296	pand xmm1,xmm12
				297	pand xmm11,xmm13
				298	pand xmm3,xmm8
				299	pxor xmm7,xmm11
				300	pxor xmm3,xmm1
				301	pxor xmm11,xmm10
				302	pxor xmm1,xmm9
				303	pxor xmm14,xmm12
				304	pxor xmm13,xmm8
				305	movdqa xmm10,xmm14
				306	pxor xmm10,xmm13
				307	pand xmm10,xmm6
				308	pxor xmm6,xmm0
				309	pand xmm0,xmm14
				310	pand xmm6,xmm13
				311	pxor xmm6,xmm0
				312	pxor xmm0,xmm10
				313	pxor xmm6,xmm11
				314	pxor xmm3,xmm11
				315	pxor xmm0,xmm7
				316	pxor xmm1,xmm7
				317	pxor xmm6,xmm15
				318	pxor xmm0,xmm5
				319	pxor xmm3,xmm6
				320	pxor xmm5,xmm15
				321	pxor xmm15,xmm0
				322
				323	pxor xmm0,xmm4
				324	pxor xmm4,xmm1
				325	pxor xmm1,xmm2
				326	pxor xmm2,xmm4
				327	pxor xmm3,xmm4
				328
				329	pxor xmm5,xmm2
				330	dec r10d
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	331	jl NEAR $L$enc_done
				332	pshufd xmm7,xmm15,0x93
				333	pshufd xmm8,xmm0,0x93
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	334	pxor xmm15,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	335	pshufd xmm9,xmm3,0x93
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	336	pxor xmm0,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	337	pshufd xmm10,xmm5,0x93
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	338	pxor xmm3,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	339	pshufd xmm11,xmm2,0x93
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	340	pxor xmm5,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	341	pshufd xmm12,xmm6,0x93
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	342	pxor xmm2,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	343	pshufd xmm13,xmm1,0x93
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	344	pxor xmm6,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	345	pshufd xmm14,xmm4,0x93
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	346	pxor xmm1,xmm13
				347	pxor xmm4,xmm14
				348
				349	pxor xmm8,xmm15
				350	pxor xmm7,xmm4
				351	pxor xmm8,xmm4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	352	pshufd xmm15,xmm15,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	353	pxor xmm9,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	354	pshufd xmm0,xmm0,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	355	pxor xmm12,xmm2
				356	pxor xmm15,xmm7
				357	pxor xmm13,xmm6
				358	pxor xmm0,xmm8
				359	pxor xmm11,xmm5
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	360	pshufd xmm7,xmm2,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	361	pxor xmm14,xmm1
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	362	pshufd xmm8,xmm6,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	363	pxor xmm10,xmm3
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	364	pshufd xmm2,xmm5,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	365	pxor xmm10,xmm4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	366	pshufd xmm6,xmm4,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	367	pxor xmm11,xmm4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	368	pshufd xmm5,xmm1,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	369	pxor xmm7,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	370	pshufd xmm1,xmm3,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	371	pxor xmm8,xmm12
				372	pxor xmm2,xmm10
				373	pxor xmm6,xmm14
				374	pxor xmm5,xmm13
				375	movdqa xmm3,xmm7
				376	pxor xmm1,xmm9
				377	movdqa xmm4,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	378	movdqa xmm7,XMMWORD[48+r11]
				379	jnz NEAR $L$enc_loop
				380	movdqa xmm7,XMMWORD[64+r11]
				381	jmp NEAR $L$enc_loop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	382	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	383	$L$enc_done:
				384	movdqa xmm7,XMMWORD[r11]
				385	movdqa xmm8,XMMWORD[16+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	386	movdqa xmm9,xmm1
				387	psrlq xmm1,1
				388	movdqa xmm10,xmm2
				389	psrlq xmm2,1
				390	pxor xmm1,xmm4
				391	pxor xmm2,xmm6
				392	pand xmm1,xmm7
				393	pand xmm2,xmm7
				394	pxor xmm4,xmm1
				395	psllq xmm1,1
				396	pxor xmm6,xmm2
				397	psllq xmm2,1
				398	pxor xmm1,xmm9
				399	pxor xmm2,xmm10
				400	movdqa xmm9,xmm3
				401	psrlq xmm3,1
				402	movdqa xmm10,xmm15
				403	psrlq xmm15,1
				404	pxor xmm3,xmm5
				405	pxor xmm15,xmm0
				406	pand xmm3,xmm7
				407	pand xmm15,xmm7
				408	pxor xmm5,xmm3
				409	psllq xmm3,1
				410	pxor xmm0,xmm15
				411	psllq xmm15,1
				412	pxor xmm3,xmm9
				413	pxor xmm15,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	414	movdqa xmm7,XMMWORD[32+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	415	movdqa xmm9,xmm6
				416	psrlq xmm6,2
				417	movdqa xmm10,xmm2
				418	psrlq xmm2,2
				419	pxor xmm6,xmm4
				420	pxor xmm2,xmm1
				421	pand xmm6,xmm8
				422	pand xmm2,xmm8
				423	pxor xmm4,xmm6
				424	psllq xmm6,2
				425	pxor xmm1,xmm2
				426	psllq xmm2,2
				427	pxor xmm6,xmm9
				428	pxor xmm2,xmm10
				429	movdqa xmm9,xmm0
				430	psrlq xmm0,2
				431	movdqa xmm10,xmm15
				432	psrlq xmm15,2
				433	pxor xmm0,xmm5
				434	pxor xmm15,xmm3
				435	pand xmm0,xmm8
				436	pand xmm15,xmm8
				437	pxor xmm5,xmm0
				438	psllq xmm0,2
				439	pxor xmm3,xmm15
				440	psllq xmm15,2
				441	pxor xmm0,xmm9
				442	pxor xmm15,xmm10
				443	movdqa xmm9,xmm5
				444	psrlq xmm5,4
				445	movdqa xmm10,xmm3
				446	psrlq xmm3,4
				447	pxor xmm5,xmm4
				448	pxor xmm3,xmm1
				449	pand xmm5,xmm7
				450	pand xmm3,xmm7
				451	pxor xmm4,xmm5
				452	psllq xmm5,4
				453	pxor xmm1,xmm3
				454	psllq xmm3,4
				455	pxor xmm5,xmm9
				456	pxor xmm3,xmm10
				457	movdqa xmm9,xmm0
				458	psrlq xmm0,4
				459	movdqa xmm10,xmm15
				460	psrlq xmm15,4
				461	pxor xmm0,xmm6
				462	pxor xmm15,xmm2
				463	pand xmm0,xmm7
				464	pand xmm15,xmm7
				465	pxor xmm6,xmm0
				466	psllq xmm0,4
				467	pxor xmm2,xmm15
				468	psllq xmm15,4
				469	pxor xmm0,xmm9
				470	pxor xmm15,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	471	movdqa xmm7,XMMWORD[rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	472	pxor xmm3,xmm7
				473	pxor xmm5,xmm7
				474	pxor xmm2,xmm7
				475	pxor xmm6,xmm7
				476	pxor xmm1,xmm7
				477	pxor xmm4,xmm7
				478	pxor xmm15,xmm7
				479	pxor xmm0,xmm7
				480	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	481
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	482
				483
				484	ALIGN 64
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	485	_bsaes_decrypt8:
				486	lea r11,[$L$BS0]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	487
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	488	movdqa xmm8,XMMWORD[rax]
				489	lea rax,[16+rax]
				490	movdqa xmm7,XMMWORD[((-48))+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	491	pxor xmm15,xmm8
				492	pxor xmm0,xmm8
				493	pxor xmm1,xmm8
				494	pxor xmm2,xmm8
				495	DB 102,68,15,56,0,255
				496	DB 102,15,56,0,199
				497	pxor xmm3,xmm8
				498	pxor xmm4,xmm8
				499	DB 102,15,56,0,207
				500	DB 102,15,56,0,215
				501	pxor xmm5,xmm8
				502	pxor xmm6,xmm8
				503	DB 102,15,56,0,223
				504	DB 102,15,56,0,231
				505	DB 102,15,56,0,239
				506	DB 102,15,56,0,247
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	507	movdqa xmm7,XMMWORD[r11]
				508	movdqa xmm8,XMMWORD[16+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	509	movdqa xmm9,xmm5
				510	psrlq xmm5,1
				511	movdqa xmm10,xmm3
				512	psrlq xmm3,1
				513	pxor xmm5,xmm6
				514	pxor xmm3,xmm4
				515	pand xmm5,xmm7
				516	pand xmm3,xmm7
				517	pxor xmm6,xmm5
				518	psllq xmm5,1
				519	pxor xmm4,xmm3
				520	psllq xmm3,1
				521	pxor xmm5,xmm9
				522	pxor xmm3,xmm10
				523	movdqa xmm9,xmm1
				524	psrlq xmm1,1
				525	movdqa xmm10,xmm15
				526	psrlq xmm15,1
				527	pxor xmm1,xmm2
				528	pxor xmm15,xmm0
				529	pand xmm1,xmm7
				530	pand xmm15,xmm7
				531	pxor xmm2,xmm1
				532	psllq xmm1,1
				533	pxor xmm0,xmm15
				534	psllq xmm15,1
				535	pxor xmm1,xmm9
				536	pxor xmm15,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	537	movdqa xmm7,XMMWORD[32+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	538	movdqa xmm9,xmm4
				539	psrlq xmm4,2
				540	movdqa xmm10,xmm3
				541	psrlq xmm3,2
				542	pxor xmm4,xmm6
				543	pxor xmm3,xmm5
				544	pand xmm4,xmm8
				545	pand xmm3,xmm8
				546	pxor xmm6,xmm4
				547	psllq xmm4,2
				548	pxor xmm5,xmm3
				549	psllq xmm3,2
				550	pxor xmm4,xmm9
				551	pxor xmm3,xmm10
				552	movdqa xmm9,xmm0
				553	psrlq xmm0,2
				554	movdqa xmm10,xmm15
				555	psrlq xmm15,2
				556	pxor xmm0,xmm2
				557	pxor xmm15,xmm1
				558	pand xmm0,xmm8
				559	pand xmm15,xmm8
				560	pxor xmm2,xmm0
				561	psllq xmm0,2
				562	pxor xmm1,xmm15
				563	psllq xmm15,2
				564	pxor xmm0,xmm9
				565	pxor xmm15,xmm10
				566	movdqa xmm9,xmm2
				567	psrlq xmm2,4
				568	movdqa xmm10,xmm1
				569	psrlq xmm1,4
				570	pxor xmm2,xmm6
				571	pxor xmm1,xmm5
				572	pand xmm2,xmm7
				573	pand xmm1,xmm7
				574	pxor xmm6,xmm2
				575	psllq xmm2,4
				576	pxor xmm5,xmm1
				577	psllq xmm1,4
				578	pxor xmm2,xmm9
				579	pxor xmm1,xmm10
				580	movdqa xmm9,xmm0
				581	psrlq xmm0,4
				582	movdqa xmm10,xmm15
				583	psrlq xmm15,4
				584	pxor xmm0,xmm4
				585	pxor xmm15,xmm3
				586	pand xmm0,xmm7
				587	pand xmm15,xmm7
				588	pxor xmm4,xmm0
				589	psllq xmm0,4
				590	pxor xmm3,xmm15
				591	psllq xmm15,4
				592	pxor xmm0,xmm9
				593	pxor xmm15,xmm10
				594	dec r10d
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	595	jmp NEAR $L$dec_sbox
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	596	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	597	$L$dec_loop:
				598	pxor xmm15,XMMWORD[rax]
				599	pxor xmm0,XMMWORD[16+rax]
				600	pxor xmm1,XMMWORD[32+rax]
				601	pxor xmm2,XMMWORD[48+rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	602	DB 102,68,15,56,0,255
				603	DB 102,15,56,0,199
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	604	pxor xmm3,XMMWORD[64+rax]
				605	pxor xmm4,XMMWORD[80+rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	606	DB 102,15,56,0,207
				607	DB 102,15,56,0,215
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	608	pxor xmm5,XMMWORD[96+rax]
				609	pxor xmm6,XMMWORD[112+rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	610	DB 102,15,56,0,223
				611	DB 102,15,56,0,231
				612	DB 102,15,56,0,239
				613	DB 102,15,56,0,247
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	614	lea rax,[128+rax]
				615	$L$dec_sbox:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	616	pxor xmm2,xmm3
				617
				618	pxor xmm3,xmm6
				619	pxor xmm1,xmm6
				620	pxor xmm5,xmm3
				621	pxor xmm6,xmm5
				622	pxor xmm0,xmm6
				623
				624	pxor xmm15,xmm0
				625	pxor xmm1,xmm4
				626	pxor xmm2,xmm15
				627	pxor xmm4,xmm15
				628	pxor xmm0,xmm2
				629	movdqa xmm10,xmm2
				630	movdqa xmm9,xmm6
				631	movdqa xmm8,xmm0
				632	movdqa xmm12,xmm3
				633	movdqa xmm11,xmm4
				634
				635	pxor xmm10,xmm15
				636	pxor xmm9,xmm3
				637	pxor xmm8,xmm5
				638	movdqa xmm13,xmm10
				639	pxor xmm12,xmm15
				640	movdqa xmm7,xmm9
				641	pxor xmm11,xmm1
				642	movdqa xmm14,xmm10
				643
				644	por xmm9,xmm8
				645	por xmm10,xmm11
				646	pxor xmm14,xmm7
				647	pand xmm13,xmm11
				648	pxor xmm11,xmm8
				649	pand xmm7,xmm8
				650	pand xmm14,xmm11
				651	movdqa xmm11,xmm5
				652	pxor xmm11,xmm1
				653	pand xmm12,xmm11
				654	pxor xmm10,xmm12
				655	pxor xmm9,xmm12
				656	movdqa xmm12,xmm2
				657	movdqa xmm11,xmm0
				658	pxor xmm12,xmm6
				659	pxor xmm11,xmm4
				660	movdqa xmm8,xmm12
				661	pand xmm12,xmm11
				662	por xmm8,xmm11
				663	pxor xmm7,xmm12
				664	pxor xmm10,xmm14
				665	pxor xmm9,xmm13
				666	pxor xmm8,xmm14
				667	movdqa xmm11,xmm3
				668	pxor xmm7,xmm13
				669	movdqa xmm12,xmm15
				670	pxor xmm8,xmm13
				671	movdqa xmm13,xmm6
				672	pand xmm11,xmm5
				673	movdqa xmm14,xmm2
				674	pand xmm12,xmm1
				675	pand xmm13,xmm0
				676	por xmm14,xmm4
				677	pxor xmm10,xmm11
				678	pxor xmm9,xmm12
				679	pxor xmm8,xmm13
				680	pxor xmm7,xmm14
				681
				682
				683
				684
				685
				686	movdqa xmm11,xmm10
				687	pand xmm10,xmm8
				688	pxor xmm11,xmm9
				689
				690	movdqa xmm13,xmm7
				691	movdqa xmm14,xmm11
				692	pxor xmm13,xmm10
				693	pand xmm14,xmm13
				694
				695	movdqa xmm12,xmm8
				696	pxor xmm14,xmm9
				697	pxor xmm12,xmm7
				698
				699	pxor xmm10,xmm9
				700
				701	pand xmm12,xmm10
				702
				703	movdqa xmm9,xmm13
				704	pxor xmm12,xmm7
				705
				706	pxor xmm9,xmm12
				707	pxor xmm8,xmm12
				708
				709	pand xmm9,xmm7
				710
				711	pxor xmm13,xmm9
				712	pxor xmm8,xmm9
				713
				714	pand xmm13,xmm14
				715
				716	pxor xmm13,xmm11
				717	movdqa xmm11,xmm4
				718	movdqa xmm7,xmm0
				719	movdqa xmm9,xmm14
				720	pxor xmm9,xmm13
				721	pand xmm9,xmm4
				722	pxor xmm4,xmm0
				723	pand xmm0,xmm14
				724	pand xmm4,xmm13
				725	pxor xmm4,xmm0
				726	pxor xmm0,xmm9
				727	pxor xmm11,xmm1
				728	pxor xmm7,xmm5
				729	pxor xmm14,xmm12
				730	pxor xmm13,xmm8
				731	movdqa xmm10,xmm14
				732	movdqa xmm9,xmm12
				733	pxor xmm10,xmm13
				734	pxor xmm9,xmm8
				735	pand xmm10,xmm11
				736	pand xmm9,xmm1
				737	pxor xmm11,xmm7
				738	pxor xmm1,xmm5
				739	pand xmm7,xmm14
				740	pand xmm5,xmm12
				741	pand xmm11,xmm13
				742	pand xmm1,xmm8
				743	pxor xmm7,xmm11
				744	pxor xmm1,xmm5
				745	pxor xmm11,xmm10
				746	pxor xmm5,xmm9
				747	pxor xmm4,xmm11
				748	pxor xmm1,xmm11
				749	pxor xmm0,xmm7
				750	pxor xmm5,xmm7
				751
				752	movdqa xmm11,xmm2
				753	movdqa xmm7,xmm6
				754	pxor xmm11,xmm15
				755	pxor xmm7,xmm3
				756	movdqa xmm10,xmm14
				757	movdqa xmm9,xmm12
				758	pxor xmm10,xmm13
				759	pxor xmm9,xmm8
				760	pand xmm10,xmm11
				761	pand xmm9,xmm15
				762	pxor xmm11,xmm7
				763	pxor xmm15,xmm3
				764	pand xmm7,xmm14
				765	pand xmm3,xmm12
				766	pand xmm11,xmm13
				767	pand xmm15,xmm8
				768	pxor xmm7,xmm11
				769	pxor xmm15,xmm3
				770	pxor xmm11,xmm10
				771	pxor xmm3,xmm9
				772	pxor xmm14,xmm12
				773	pxor xmm13,xmm8
				774	movdqa xmm10,xmm14
				775	pxor xmm10,xmm13
				776	pand xmm10,xmm2
				777	pxor xmm2,xmm6
				778	pand xmm6,xmm14
				779	pand xmm2,xmm13
				780	pxor xmm2,xmm6
				781	pxor xmm6,xmm10
				782	pxor xmm2,xmm11
				783	pxor xmm15,xmm11
				784	pxor xmm6,xmm7
				785	pxor xmm3,xmm7
				786	pxor xmm0,xmm6
				787	pxor xmm5,xmm4
				788
				789	pxor xmm3,xmm0
				790	pxor xmm1,xmm6
				791	pxor xmm4,xmm6
				792	pxor xmm3,xmm1
				793	pxor xmm6,xmm15
				794	pxor xmm3,xmm4
				795	pxor xmm2,xmm5
				796	pxor xmm5,xmm0
				797	pxor xmm2,xmm3
				798
				799	pxor xmm3,xmm15
				800	pxor xmm6,xmm2
				801	dec r10d
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	802	jl NEAR $L$dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	803
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	804	pshufd xmm7,xmm15,0x4E
				805	pshufd xmm13,xmm2,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	806	pxor xmm7,xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	807	pshufd xmm14,xmm4,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	808	pxor xmm13,xmm2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	809	pshufd xmm8,xmm0,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	810	pxor xmm14,xmm4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	811	pshufd xmm9,xmm5,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	812	pxor xmm8,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	813	pshufd xmm10,xmm3,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	814	pxor xmm9,xmm5
				815	pxor xmm15,xmm13
				816	pxor xmm0,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	817	pshufd xmm11,xmm1,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	818	pxor xmm10,xmm3
				819	pxor xmm5,xmm7
				820	pxor xmm3,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	821	pshufd xmm12,xmm6,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	822	pxor xmm11,xmm1
				823	pxor xmm0,xmm14
				824	pxor xmm1,xmm9
				825	pxor xmm12,xmm6
				826
				827	pxor xmm5,xmm14
				828	pxor xmm3,xmm13
				829	pxor xmm1,xmm13
				830	pxor xmm6,xmm10
				831	pxor xmm2,xmm11
				832	pxor xmm1,xmm14
				833	pxor xmm6,xmm14
				834	pxor xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	835	pshufd xmm7,xmm15,0x93
				836	pshufd xmm8,xmm0,0x93
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	837	pxor xmm15,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	838	pshufd xmm9,xmm5,0x93
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	839	pxor xmm0,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	840	pshufd xmm10,xmm3,0x93
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	841	pxor xmm5,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	842	pshufd xmm11,xmm1,0x93
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	843	pxor xmm3,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	844	pshufd xmm12,xmm6,0x93
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	845	pxor xmm1,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	846	pshufd xmm13,xmm2,0x93
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	847	pxor xmm6,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	848	pshufd xmm14,xmm4,0x93
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	849	pxor xmm2,xmm13
				850	pxor xmm4,xmm14
				851
				852	pxor xmm8,xmm15
				853	pxor xmm7,xmm4
				854	pxor xmm8,xmm4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	855	pshufd xmm15,xmm15,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	856	pxor xmm9,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	857	pshufd xmm0,xmm0,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	858	pxor xmm12,xmm1
				859	pxor xmm15,xmm7
				860	pxor xmm13,xmm6
				861	pxor xmm0,xmm8
				862	pxor xmm11,xmm3
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	863	pshufd xmm7,xmm1,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	864	pxor xmm14,xmm2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	865	pshufd xmm8,xmm6,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	866	pxor xmm10,xmm5
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	867	pshufd xmm1,xmm3,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	868	pxor xmm10,xmm4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	869	pshufd xmm6,xmm4,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	870	pxor xmm11,xmm4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	871	pshufd xmm3,xmm2,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	872	pxor xmm7,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	873	pshufd xmm2,xmm5,0x4E
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	874	pxor xmm8,xmm12
				875	pxor xmm10,xmm1
				876	pxor xmm6,xmm14
				877	pxor xmm13,xmm3
				878	movdqa xmm3,xmm7
				879	pxor xmm2,xmm9
				880	movdqa xmm5,xmm13
				881	movdqa xmm4,xmm8
				882	movdqa xmm1,xmm2
				883	movdqa xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	884	movdqa xmm7,XMMWORD[((-16))+r11]
				885	jnz NEAR $L$dec_loop
				886	movdqa xmm7,XMMWORD[((-32))+r11]
				887	jmp NEAR $L$dec_loop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	888	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	889	$L$dec_done:
				890	movdqa xmm7,XMMWORD[r11]
				891	movdqa xmm8,XMMWORD[16+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	892	movdqa xmm9,xmm2
				893	psrlq xmm2,1
				894	movdqa xmm10,xmm1
				895	psrlq xmm1,1
				896	pxor xmm2,xmm4
				897	pxor xmm1,xmm6
				898	pand xmm2,xmm7
				899	pand xmm1,xmm7
				900	pxor xmm4,xmm2
				901	psllq xmm2,1
				902	pxor xmm6,xmm1
				903	psllq xmm1,1
				904	pxor xmm2,xmm9
				905	pxor xmm1,xmm10
				906	movdqa xmm9,xmm5
				907	psrlq xmm5,1
				908	movdqa xmm10,xmm15
				909	psrlq xmm15,1
				910	pxor xmm5,xmm3
				911	pxor xmm15,xmm0
				912	pand xmm5,xmm7
				913	pand xmm15,xmm7
				914	pxor xmm3,xmm5
				915	psllq xmm5,1
				916	pxor xmm0,xmm15
				917	psllq xmm15,1
				918	pxor xmm5,xmm9
				919	pxor xmm15,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	920	movdqa xmm7,XMMWORD[32+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	921	movdqa xmm9,xmm6
				922	psrlq xmm6,2
				923	movdqa xmm10,xmm1
				924	psrlq xmm1,2
				925	pxor xmm6,xmm4
				926	pxor xmm1,xmm2
				927	pand xmm6,xmm8
				928	pand xmm1,xmm8
				929	pxor xmm4,xmm6
				930	psllq xmm6,2
				931	pxor xmm2,xmm1
				932	psllq xmm1,2
				933	pxor xmm6,xmm9
				934	pxor xmm1,xmm10
				935	movdqa xmm9,xmm0
				936	psrlq xmm0,2
				937	movdqa xmm10,xmm15
				938	psrlq xmm15,2
				939	pxor xmm0,xmm3
				940	pxor xmm15,xmm5
				941	pand xmm0,xmm8
				942	pand xmm15,xmm8
				943	pxor xmm3,xmm0
				944	psllq xmm0,2
				945	pxor xmm5,xmm15
				946	psllq xmm15,2
				947	pxor xmm0,xmm9
				948	pxor xmm15,xmm10
				949	movdqa xmm9,xmm3
				950	psrlq xmm3,4
				951	movdqa xmm10,xmm5
				952	psrlq xmm5,4
				953	pxor xmm3,xmm4
				954	pxor xmm5,xmm2
				955	pand xmm3,xmm7
				956	pand xmm5,xmm7
				957	pxor xmm4,xmm3
				958	psllq xmm3,4
				959	pxor xmm2,xmm5
				960	psllq xmm5,4
				961	pxor xmm3,xmm9
				962	pxor xmm5,xmm10
				963	movdqa xmm9,xmm0
				964	psrlq xmm0,4
				965	movdqa xmm10,xmm15
				966	psrlq xmm15,4
				967	pxor xmm0,xmm6
				968	pxor xmm15,xmm1
				969	pand xmm0,xmm7
				970	pand xmm15,xmm7
				971	pxor xmm6,xmm0
				972	psllq xmm0,4
				973	pxor xmm1,xmm15
				974	psllq xmm15,4
				975	pxor xmm0,xmm9
				976	pxor xmm15,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	977	movdqa xmm7,XMMWORD[rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	978	pxor xmm5,xmm7
				979	pxor xmm3,xmm7
				980	pxor xmm1,xmm7
				981	pxor xmm6,xmm7
				982	pxor xmm2,xmm7
				983	pxor xmm4,xmm7
				984	pxor xmm15,xmm7
				985	pxor xmm0,xmm7
				986	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	987
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	988
				989	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	990	_bsaes_key_convert:
				991	lea r11,[$L$masks]
				992	movdqu xmm7,XMMWORD[rcx]
				993	lea rcx,[16+rcx]
				994	movdqa xmm0,XMMWORD[r11]
				995	movdqa xmm1,XMMWORD[16+r11]
				996	movdqa xmm2,XMMWORD[32+r11]
				997	movdqa xmm3,XMMWORD[48+r11]
				998	movdqa xmm4,XMMWORD[64+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	999	pcmpeqd xmm5,xmm5
				1000
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1001	movdqu xmm6,XMMWORD[rcx]
				1002	movdqa XMMWORD[rax],xmm7
				1003	lea rax,[16+rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1004	dec r10d
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1005	jmp NEAR $L$key_loop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1006	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1007	$L$key_loop:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1008	DB 102,15,56,0,244
				1009
				1010	movdqa xmm8,xmm0
				1011	movdqa xmm9,xmm1
				1012
				1013	pand xmm8,xmm6
				1014	pand xmm9,xmm6
				1015	movdqa xmm10,xmm2
				1016	pcmpeqb xmm8,xmm0
				1017	psllq xmm0,4
				1018	movdqa xmm11,xmm3
				1019	pcmpeqb xmm9,xmm1
				1020	psllq xmm1,4
				1021
				1022	pand xmm10,xmm6
				1023	pand xmm11,xmm6
				1024	movdqa xmm12,xmm0
				1025	pcmpeqb xmm10,xmm2
				1026	psllq xmm2,4
				1027	movdqa xmm13,xmm1
				1028	pcmpeqb xmm11,xmm3
				1029	psllq xmm3,4
				1030
				1031	movdqa xmm14,xmm2
				1032	movdqa xmm15,xmm3
				1033	pxor xmm8,xmm5
				1034	pxor xmm9,xmm5
				1035
				1036	pand xmm12,xmm6
				1037	pand xmm13,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1038	movdqa XMMWORD[rax],xmm8
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1039	pcmpeqb xmm12,xmm0
				1040	psrlq xmm0,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1041	movdqa XMMWORD[16+rax],xmm9
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1042	pcmpeqb xmm13,xmm1
				1043	psrlq xmm1,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1044	lea rcx,[16+rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1045
				1046	pand xmm14,xmm6
				1047	pand xmm15,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1048	movdqa XMMWORD[32+rax],xmm10
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1049	pcmpeqb xmm14,xmm2
				1050	psrlq xmm2,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1051	movdqa XMMWORD[48+rax],xmm11
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1052	pcmpeqb xmm15,xmm3
				1053	psrlq xmm3,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1054	movdqu xmm6,XMMWORD[rcx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1055
				1056	pxor xmm13,xmm5
				1057	pxor xmm14,xmm5
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1058	movdqa XMMWORD[64+rax],xmm12
				1059	movdqa XMMWORD[80+rax],xmm13
				1060	movdqa XMMWORD[96+rax],xmm14
				1061	movdqa XMMWORD[112+rax],xmm15
				1062	lea rax,[128+rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1063	dec r10d
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1064	jnz NEAR $L$key_loop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1065
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1066	movdqa xmm7,XMMWORD[80+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1067
				1068	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1069
				1070	EXTERN asm_AES_cbc_encrypt
				1071	global bsaes_cbc_encrypt
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1072
				1073	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1074	bsaes_cbc_encrypt:
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1075
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1076	mov r11d,DWORD[48+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1077	cmp r11d,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1078	jne NEAR asm_AES_cbc_encrypt
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1079	cmp r8,128
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1080	jb NEAR asm_AES_cbc_encrypt
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1081
				1082	mov rax,rsp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1083	$L$cbc_dec_prologue:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1084	push rbp
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1085
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1086	push rbx
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1087
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1088	push r12
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1089
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1090	push r13
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1091
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1092	push r14
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1093
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1094	push r15
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1095
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1096	lea rsp,[((-72))+rsp]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1097
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1098	mov r10,QWORD[160+rsp]
				1099	lea rsp,[((-160))+rsp]
				1100	movaps XMMWORD[64+rsp],xmm6
				1101	movaps XMMWORD[80+rsp],xmm7
				1102	movaps XMMWORD[96+rsp],xmm8
				1103	movaps XMMWORD[112+rsp],xmm9
				1104	movaps XMMWORD[128+rsp],xmm10
				1105	movaps XMMWORD[144+rsp],xmm11
				1106	movaps XMMWORD[160+rsp],xmm12
				1107	movaps XMMWORD[176+rsp],xmm13
				1108	movaps XMMWORD[192+rsp],xmm14
				1109	movaps XMMWORD[208+rsp],xmm15
				1110	$L$cbc_dec_body:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1111	mov rbp,rsp
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1112
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1113	mov eax,DWORD[240+r9]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1114	mov r12,rcx
				1115	mov r13,rdx
				1116	mov r14,r8
				1117	mov r15,r9
				1118	mov rbx,r10
				1119	shr r14,4
				1120
				1121	mov edx,eax
				1122	shl rax,7
				1123	sub rax,96
				1124	sub rsp,rax
				1125
				1126	mov rax,rsp
				1127	mov rcx,r15
				1128	mov r10d,edx
				1129	call _bsaes_key_convert
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1130	pxor xmm7,XMMWORD[rsp]
				1131	movdqa XMMWORD[rax],xmm6
				1132	movdqa XMMWORD[rsp],xmm7
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1133
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1134	movdqu xmm14,XMMWORD[rbx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1135	sub r14,8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1136	$L$cbc_dec_loop:
				1137	movdqu xmm15,XMMWORD[r12]
				1138	movdqu xmm0,XMMWORD[16+r12]
				1139	movdqu xmm1,XMMWORD[32+r12]
				1140	movdqu xmm2,XMMWORD[48+r12]
				1141	movdqu xmm3,XMMWORD[64+r12]
				1142	movdqu xmm4,XMMWORD[80+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1143	mov rax,rsp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1144	movdqu xmm5,XMMWORD[96+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1145	mov r10d,edx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1146	movdqu xmm6,XMMWORD[112+r12]
				1147	movdqa XMMWORD[32+rbp],xmm14
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1148
				1149	call _bsaes_decrypt8
				1150
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1151	pxor xmm15,XMMWORD[32+rbp]
				1152	movdqu xmm7,XMMWORD[r12]
				1153	movdqu xmm8,XMMWORD[16+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1154	pxor xmm0,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1155	movdqu xmm9,XMMWORD[32+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1156	pxor xmm5,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1157	movdqu xmm10,XMMWORD[48+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1158	pxor xmm3,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1159	movdqu xmm11,XMMWORD[64+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1160	pxor xmm1,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1161	movdqu xmm12,XMMWORD[80+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1162	pxor xmm6,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1163	movdqu xmm13,XMMWORD[96+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1164	pxor xmm2,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1165	movdqu xmm14,XMMWORD[112+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1166	pxor xmm4,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1167	movdqu XMMWORD[r13],xmm15
				1168	lea r12,[128+r12]
				1169	movdqu XMMWORD[16+r13],xmm0
				1170	movdqu XMMWORD[32+r13],xmm5
				1171	movdqu XMMWORD[48+r13],xmm3
				1172	movdqu XMMWORD[64+r13],xmm1
				1173	movdqu XMMWORD[80+r13],xmm6
				1174	movdqu XMMWORD[96+r13],xmm2
				1175	movdqu XMMWORD[112+r13],xmm4
				1176	lea r13,[128+r13]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1177	sub r14,8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1178	jnc NEAR $L$cbc_dec_loop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1179
				1180	add r14,8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1181	jz NEAR $L$cbc_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1182
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1183	movdqu xmm15,XMMWORD[r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1184	mov rax,rsp
				1185	mov r10d,edx
				1186	cmp r14,2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1187	jb NEAR $L$cbc_dec_one
				1188	movdqu xmm0,XMMWORD[16+r12]
				1189	je NEAR $L$cbc_dec_two
				1190	movdqu xmm1,XMMWORD[32+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1191	cmp r14,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1192	jb NEAR $L$cbc_dec_three
				1193	movdqu xmm2,XMMWORD[48+r12]
				1194	je NEAR $L$cbc_dec_four
				1195	movdqu xmm3,XMMWORD[64+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1196	cmp r14,6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1197	jb NEAR $L$cbc_dec_five
				1198	movdqu xmm4,XMMWORD[80+r12]
				1199	je NEAR $L$cbc_dec_six
				1200	movdqu xmm5,XMMWORD[96+r12]
				1201	movdqa XMMWORD[32+rbp],xmm14
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1202	call _bsaes_decrypt8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1203	pxor xmm15,XMMWORD[32+rbp]
				1204	movdqu xmm7,XMMWORD[r12]
				1205	movdqu xmm8,XMMWORD[16+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1206	pxor xmm0,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1207	movdqu xmm9,XMMWORD[32+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1208	pxor xmm5,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1209	movdqu xmm10,XMMWORD[48+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1210	pxor xmm3,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1211	movdqu xmm11,XMMWORD[64+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1212	pxor xmm1,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1213	movdqu xmm12,XMMWORD[80+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1214	pxor xmm6,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1215	movdqu xmm14,XMMWORD[96+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1216	pxor xmm2,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1217	movdqu XMMWORD[r13],xmm15
				1218	movdqu XMMWORD[16+r13],xmm0
				1219	movdqu XMMWORD[32+r13],xmm5
				1220	movdqu XMMWORD[48+r13],xmm3
				1221	movdqu XMMWORD[64+r13],xmm1
				1222	movdqu XMMWORD[80+r13],xmm6
				1223	movdqu XMMWORD[96+r13],xmm2
				1224	jmp NEAR $L$cbc_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1225	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1226	$L$cbc_dec_six:
				1227	movdqa XMMWORD[32+rbp],xmm14
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1228	call _bsaes_decrypt8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1229	pxor xmm15,XMMWORD[32+rbp]
				1230	movdqu xmm7,XMMWORD[r12]
				1231	movdqu xmm8,XMMWORD[16+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1232	pxor xmm0,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1233	movdqu xmm9,XMMWORD[32+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1234	pxor xmm5,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1235	movdqu xmm10,XMMWORD[48+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1236	pxor xmm3,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1237	movdqu xmm11,XMMWORD[64+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1238	pxor xmm1,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1239	movdqu xmm14,XMMWORD[80+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1240	pxor xmm6,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1241	movdqu XMMWORD[r13],xmm15
				1242	movdqu XMMWORD[16+r13],xmm0
				1243	movdqu XMMWORD[32+r13],xmm5
				1244	movdqu XMMWORD[48+r13],xmm3
				1245	movdqu XMMWORD[64+r13],xmm1
				1246	movdqu XMMWORD[80+r13],xmm6
				1247	jmp NEAR $L$cbc_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1248	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1249	$L$cbc_dec_five:
				1250	movdqa XMMWORD[32+rbp],xmm14
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1251	call _bsaes_decrypt8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1252	pxor xmm15,XMMWORD[32+rbp]
				1253	movdqu xmm7,XMMWORD[r12]
				1254	movdqu xmm8,XMMWORD[16+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1255	pxor xmm0,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1256	movdqu xmm9,XMMWORD[32+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1257	pxor xmm5,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1258	movdqu xmm10,XMMWORD[48+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1259	pxor xmm3,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1260	movdqu xmm14,XMMWORD[64+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1261	pxor xmm1,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1262	movdqu XMMWORD[r13],xmm15
				1263	movdqu XMMWORD[16+r13],xmm0
				1264	movdqu XMMWORD[32+r13],xmm5
				1265	movdqu XMMWORD[48+r13],xmm3
				1266	movdqu XMMWORD[64+r13],xmm1
				1267	jmp NEAR $L$cbc_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1268	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1269	$L$cbc_dec_four:
				1270	movdqa XMMWORD[32+rbp],xmm14
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1271	call _bsaes_decrypt8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1272	pxor xmm15,XMMWORD[32+rbp]
				1273	movdqu xmm7,XMMWORD[r12]
				1274	movdqu xmm8,XMMWORD[16+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1275	pxor xmm0,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1276	movdqu xmm9,XMMWORD[32+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1277	pxor xmm5,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1278	movdqu xmm14,XMMWORD[48+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1279	pxor xmm3,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1280	movdqu XMMWORD[r13],xmm15
				1281	movdqu XMMWORD[16+r13],xmm0
				1282	movdqu XMMWORD[32+r13],xmm5
				1283	movdqu XMMWORD[48+r13],xmm3
				1284	jmp NEAR $L$cbc_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1285	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1286	$L$cbc_dec_three:
				1287	movdqa XMMWORD[32+rbp],xmm14
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1288	call _bsaes_decrypt8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1289	pxor xmm15,XMMWORD[32+rbp]
				1290	movdqu xmm7,XMMWORD[r12]
				1291	movdqu xmm8,XMMWORD[16+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1292	pxor xmm0,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1293	movdqu xmm14,XMMWORD[32+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1294	pxor xmm5,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1295	movdqu XMMWORD[r13],xmm15
				1296	movdqu XMMWORD[16+r13],xmm0
				1297	movdqu XMMWORD[32+r13],xmm5
				1298	jmp NEAR $L$cbc_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1299	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1300	$L$cbc_dec_two:
				1301	movdqa XMMWORD[32+rbp],xmm14
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1302	call _bsaes_decrypt8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1303	pxor xmm15,XMMWORD[32+rbp]
				1304	movdqu xmm7,XMMWORD[r12]
				1305	movdqu xmm14,XMMWORD[16+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1306	pxor xmm0,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1307	movdqu XMMWORD[r13],xmm15
				1308	movdqu XMMWORD[16+r13],xmm0
				1309	jmp NEAR $L$cbc_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1310	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1311	$L$cbc_dec_one:
				1312	lea rcx,[r12]
				1313	lea rdx,[32+rbp]
				1314	lea r8,[r15]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1315	call asm_AES_decrypt
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1316	pxor xmm14,XMMWORD[32+rbp]
				1317	movdqu XMMWORD[r13],xmm14
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1318	movdqa xmm14,xmm15
				1319
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1320	$L$cbc_dec_done:
				1321	movdqu XMMWORD[rbx],xmm14
				1322	lea rax,[rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1323	pxor xmm0,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1324	$L$cbc_dec_bzero:
				1325	movdqa XMMWORD[rax],xmm0
				1326	movdqa XMMWORD[16+rax],xmm0
				1327	lea rax,[32+rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1328	cmp rbp,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1329	ja NEAR $L$cbc_dec_bzero
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1330
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1331	lea rax,[120+rbp]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1332
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1333	movaps xmm6,XMMWORD[64+rbp]
				1334	movaps xmm7,XMMWORD[80+rbp]
				1335	movaps xmm8,XMMWORD[96+rbp]
				1336	movaps xmm9,XMMWORD[112+rbp]
				1337	movaps xmm10,XMMWORD[128+rbp]
				1338	movaps xmm11,XMMWORD[144+rbp]
				1339	movaps xmm12,XMMWORD[160+rbp]
				1340	movaps xmm13,XMMWORD[176+rbp]
				1341	movaps xmm14,XMMWORD[192+rbp]
				1342	movaps xmm15,XMMWORD[208+rbp]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1343	lea rax,[160+rax]
				1344	$L$cbc_dec_tail:
				1345	mov r15,QWORD[((-48))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1346
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1347	mov r14,QWORD[((-40))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1348
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1349	mov r13,QWORD[((-32))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1350
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1351	mov r12,QWORD[((-24))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1352
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1353	mov rbx,QWORD[((-16))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1354
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1355	mov rbp,QWORD[((-8))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1356
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1357	lea rsp,[rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1358
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1359	$L$cbc_dec_epilogue:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1360	DB 0F3h,0C3h ;repret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1361
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1362
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1363
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1364	global bsaes_ctr32_encrypt_blocks
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1365
				1366	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1367	bsaes_ctr32_encrypt_blocks:
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1368
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1369	mov rax,rsp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1370	$L$ctr_enc_prologue:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1371	push rbp
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1372
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1373	push rbx
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1374
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1375	push r12
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1376
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1377	push r13
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1378
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1379	push r14
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1380
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1381	push r15
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1382
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1383	lea rsp,[((-72))+rsp]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1384
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1385	mov r10,QWORD[160+rsp]
				1386	lea rsp,[((-160))+rsp]
				1387	movaps XMMWORD[64+rsp],xmm6
				1388	movaps XMMWORD[80+rsp],xmm7
				1389	movaps XMMWORD[96+rsp],xmm8
				1390	movaps XMMWORD[112+rsp],xmm9
				1391	movaps XMMWORD[128+rsp],xmm10
				1392	movaps XMMWORD[144+rsp],xmm11
				1393	movaps XMMWORD[160+rsp],xmm12
				1394	movaps XMMWORD[176+rsp],xmm13
				1395	movaps XMMWORD[192+rsp],xmm14
				1396	movaps XMMWORD[208+rsp],xmm15
				1397	$L$ctr_enc_body:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1398	mov rbp,rsp
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1399
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1400	movdqu xmm0,XMMWORD[r10]
				1401	mov eax,DWORD[240+r9]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1402	mov r12,rcx
				1403	mov r13,rdx
				1404	mov r14,r8
				1405	mov r15,r9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1406	movdqa XMMWORD[32+rbp],xmm0
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1407	cmp r8,8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1408	jb NEAR $L$ctr_enc_short
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1409
				1410	mov ebx,eax
				1411	shl rax,7
				1412	sub rax,96
				1413	sub rsp,rax
				1414
				1415	mov rax,rsp
				1416	mov rcx,r15
				1417	mov r10d,ebx
				1418	call _bsaes_key_convert
				1419	pxor xmm7,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1420	movdqa XMMWORD[rax],xmm7
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1421
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1422	movdqa xmm8,XMMWORD[rsp]
				1423	lea r11,[$L$ADD1]
				1424	movdqa xmm15,XMMWORD[32+rbp]
				1425	movdqa xmm7,XMMWORD[((-32))+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1426	DB 102,68,15,56,0,199
				1427	DB 102,68,15,56,0,255
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1428	movdqa XMMWORD[rsp],xmm8
				1429	jmp NEAR $L$ctr_enc_loop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1430	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1431	$L$ctr_enc_loop:
				1432	movdqa XMMWORD[32+rbp],xmm15
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1433	movdqa xmm0,xmm15
				1434	movdqa xmm1,xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1435	paddd xmm0,XMMWORD[r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1436	movdqa xmm2,xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1437	paddd xmm1,XMMWORD[16+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1438	movdqa xmm3,xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1439	paddd xmm2,XMMWORD[32+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1440	movdqa xmm4,xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1441	paddd xmm3,XMMWORD[48+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1442	movdqa xmm5,xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1443	paddd xmm4,XMMWORD[64+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1444	movdqa xmm6,xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1445	paddd xmm5,XMMWORD[80+r11]
				1446	paddd xmm6,XMMWORD[96+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1447
				1448
				1449
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1450	movdqa xmm8,XMMWORD[rsp]
				1451	lea rax,[16+rsp]
				1452	movdqa xmm7,XMMWORD[((-16))+r11]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1453	pxor xmm15,xmm8
				1454	pxor xmm0,xmm8
				1455	pxor xmm1,xmm8
				1456	pxor xmm2,xmm8
				1457	DB 102,68,15,56,0,255
				1458	DB 102,15,56,0,199
				1459	pxor xmm3,xmm8
				1460	pxor xmm4,xmm8
				1461	DB 102,15,56,0,207
				1462	DB 102,15,56,0,215
				1463	pxor xmm5,xmm8
				1464	pxor xmm6,xmm8
				1465	DB 102,15,56,0,223
				1466	DB 102,15,56,0,231
				1467	DB 102,15,56,0,239
				1468	DB 102,15,56,0,247
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1469	lea r11,[$L$BS0]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1470	mov r10d,ebx
				1471
				1472	call _bsaes_encrypt8_bitslice
				1473
				1474	sub r14,8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1475	jc NEAR $L$ctr_enc_loop_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1476
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1477	movdqu xmm7,XMMWORD[r12]
				1478	movdqu xmm8,XMMWORD[16+r12]
				1479	movdqu xmm9,XMMWORD[32+r12]
				1480	movdqu xmm10,XMMWORD[48+r12]
				1481	movdqu xmm11,XMMWORD[64+r12]
				1482	movdqu xmm12,XMMWORD[80+r12]
				1483	movdqu xmm13,XMMWORD[96+r12]
				1484	movdqu xmm14,XMMWORD[112+r12]
				1485	lea r12,[128+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1486	pxor xmm7,xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1487	movdqa xmm15,XMMWORD[32+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1488	pxor xmm0,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1489	movdqu XMMWORD[r13],xmm7
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1490	pxor xmm3,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1491	movdqu XMMWORD[16+r13],xmm0
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1492	pxor xmm5,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1493	movdqu XMMWORD[32+r13],xmm3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1494	pxor xmm2,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1495	movdqu XMMWORD[48+r13],xmm5
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1496	pxor xmm6,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1497	movdqu XMMWORD[64+r13],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1498	pxor xmm1,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1499	movdqu XMMWORD[80+r13],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1500	pxor xmm4,xmm14
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1501	movdqu XMMWORD[96+r13],xmm1
				1502	lea r11,[$L$ADD1]
				1503	movdqu XMMWORD[112+r13],xmm4
				1504	lea r13,[128+r13]
				1505	paddd xmm15,XMMWORD[112+r11]
				1506	jnz NEAR $L$ctr_enc_loop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1507
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1508	jmp NEAR $L$ctr_enc_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1509	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1510	$L$ctr_enc_loop_done:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1511	add r14,8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1512	movdqu xmm7,XMMWORD[r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1513	pxor xmm15,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1514	movdqu XMMWORD[r13],xmm15
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1515	cmp r14,2
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1516	jb NEAR $L$ctr_enc_done
				1517	movdqu xmm8,XMMWORD[16+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1518	pxor xmm0,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1519	movdqu XMMWORD[16+r13],xmm0
				1520	je NEAR $L$ctr_enc_done
				1521	movdqu xmm9,XMMWORD[32+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1522	pxor xmm3,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1523	movdqu XMMWORD[32+r13],xmm3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1524	cmp r14,4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1525	jb NEAR $L$ctr_enc_done
				1526	movdqu xmm10,XMMWORD[48+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1527	pxor xmm5,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1528	movdqu XMMWORD[48+r13],xmm5
				1529	je NEAR $L$ctr_enc_done
				1530	movdqu xmm11,XMMWORD[64+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1531	pxor xmm2,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1532	movdqu XMMWORD[64+r13],xmm2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1533	cmp r14,6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1534	jb NEAR $L$ctr_enc_done
				1535	movdqu xmm12,XMMWORD[80+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1536	pxor xmm6,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1537	movdqu XMMWORD[80+r13],xmm6
				1538	je NEAR $L$ctr_enc_done
				1539	movdqu xmm13,XMMWORD[96+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1540	pxor xmm1,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1541	movdqu XMMWORD[96+r13],xmm1
				1542	jmp NEAR $L$ctr_enc_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1543
				1544	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1545	$L$ctr_enc_short:
				1546	lea rcx,[32+rbp]
				1547	lea rdx,[48+rbp]
				1548	lea r8,[r15]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1549	call asm_AES_encrypt
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1550	movdqu xmm0,XMMWORD[r12]
				1551	lea r12,[16+r12]
				1552	mov eax,DWORD[44+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1553	bswap eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1554	pxor xmm0,XMMWORD[48+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1555	inc eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1556	movdqu XMMWORD[r13],xmm0
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1557	bswap eax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1558	lea r13,[16+r13]
				1559	mov DWORD[44+rsp],eax
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1560	dec r14
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1561	jnz NEAR $L$ctr_enc_short
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1562
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1563	$L$ctr_enc_done:
				1564	lea rax,[rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1565	pxor xmm0,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1566	$L$ctr_enc_bzero:
				1567	movdqa XMMWORD[rax],xmm0
				1568	movdqa XMMWORD[16+rax],xmm0
				1569	lea rax,[32+rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1570	cmp rbp,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1571	ja NEAR $L$ctr_enc_bzero
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1572
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1573	lea rax,[120+rbp]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1574
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1575	movaps xmm6,XMMWORD[64+rbp]
				1576	movaps xmm7,XMMWORD[80+rbp]
				1577	movaps xmm8,XMMWORD[96+rbp]
				1578	movaps xmm9,XMMWORD[112+rbp]
				1579	movaps xmm10,XMMWORD[128+rbp]
				1580	movaps xmm11,XMMWORD[144+rbp]
				1581	movaps xmm12,XMMWORD[160+rbp]
				1582	movaps xmm13,XMMWORD[176+rbp]
				1583	movaps xmm14,XMMWORD[192+rbp]
				1584	movaps xmm15,XMMWORD[208+rbp]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1585	lea rax,[160+rax]
				1586	$L$ctr_enc_tail:
				1587	mov r15,QWORD[((-48))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1588
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1589	mov r14,QWORD[((-40))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1590
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1591	mov r13,QWORD[((-32))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1592
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1593	mov r12,QWORD[((-24))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1594
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1595	mov rbx,QWORD[((-16))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1596
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1597	mov rbp,QWORD[((-8))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1598
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	1599	lea rsp,[rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1600
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1601	$L$ctr_enc_epilogue:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1602	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1603
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1604
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1605	global bsaes_xts_encrypt
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1606
				1607	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1608	bsaes_xts_encrypt:
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1609
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1610	mov rax,rsp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1611	$L$xts_enc_prologue:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1612	push rbp
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1613
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1614	push rbx
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1615
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1616	push r12
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1617
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1618	push r13
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1619
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1620	push r14
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1621
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1622	push r15
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1623
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1624	lea rsp,[((-72))+rsp]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1625
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1626	mov r10,QWORD[160+rsp]
				1627	mov r11,QWORD[168+rsp]
				1628	lea rsp,[((-160))+rsp]
				1629	movaps XMMWORD[64+rsp],xmm6
				1630	movaps XMMWORD[80+rsp],xmm7
				1631	movaps XMMWORD[96+rsp],xmm8
				1632	movaps XMMWORD[112+rsp],xmm9
				1633	movaps XMMWORD[128+rsp],xmm10
				1634	movaps XMMWORD[144+rsp],xmm11
				1635	movaps XMMWORD[160+rsp],xmm12
				1636	movaps XMMWORD[176+rsp],xmm13
				1637	movaps XMMWORD[192+rsp],xmm14
				1638	movaps XMMWORD[208+rsp],xmm15
				1639	$L$xts_enc_body:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1640	mov rbp,rsp
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	1641
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1642	mov r12,rcx
				1643	mov r13,rdx
				1644	mov r14,r8
				1645	mov r15,r9
				1646
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1647	lea rcx,[r11]
				1648	lea rdx,[32+rbp]
				1649	lea r8,[r10]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1650	call asm_AES_encrypt
				1651
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1652	mov eax,DWORD[240+r15]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1653	mov rbx,r14
				1654
				1655	mov edx,eax
				1656	shl rax,7
				1657	sub rax,96
				1658	sub rsp,rax
				1659
				1660	mov rax,rsp
				1661	mov rcx,r15
				1662	mov r10d,edx
				1663	call _bsaes_key_convert
				1664	pxor xmm7,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1665	movdqa XMMWORD[rax],xmm7
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1666
				1667	and r14,-16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1668	sub rsp,0x80
				1669	movdqa xmm6,XMMWORD[32+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1670
				1671	pxor xmm14,xmm14
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1672	movdqa xmm12,XMMWORD[$L$xts_magic]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1673	pcmpgtd xmm14,xmm6
				1674
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1675	sub r14,0x80
				1676	jc NEAR $L$xts_enc_short
				1677	jmp NEAR $L$xts_enc_loop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1678
				1679	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1680	$L$xts_enc_loop:
				1681	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1682	pxor xmm14,xmm14
				1683	movdqa xmm15,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1684	movdqa XMMWORD[rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1685	paddq xmm6,xmm6
				1686	pand xmm13,xmm12
				1687	pcmpgtd xmm14,xmm6
				1688	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1689	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1690	pxor xmm14,xmm14
				1691	movdqa xmm0,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1692	movdqa XMMWORD[16+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1693	paddq xmm6,xmm6
				1694	pand xmm13,xmm12
				1695	pcmpgtd xmm14,xmm6
				1696	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1697	movdqu xmm7,XMMWORD[r12]
				1698	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1699	pxor xmm14,xmm14
				1700	movdqa xmm1,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1701	movdqa XMMWORD[32+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1702	paddq xmm6,xmm6
				1703	pand xmm13,xmm12
				1704	pcmpgtd xmm14,xmm6
				1705	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1706	movdqu xmm8,XMMWORD[16+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1707	pxor xmm15,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1708	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1709	pxor xmm14,xmm14
				1710	movdqa xmm2,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1711	movdqa XMMWORD[48+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1712	paddq xmm6,xmm6
				1713	pand xmm13,xmm12
				1714	pcmpgtd xmm14,xmm6
				1715	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1716	movdqu xmm9,XMMWORD[32+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1717	pxor xmm0,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1718	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1719	pxor xmm14,xmm14
				1720	movdqa xmm3,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1721	movdqa XMMWORD[64+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1722	paddq xmm6,xmm6
				1723	pand xmm13,xmm12
				1724	pcmpgtd xmm14,xmm6
				1725	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1726	movdqu xmm10,XMMWORD[48+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1727	pxor xmm1,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1728	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1729	pxor xmm14,xmm14
				1730	movdqa xmm4,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1731	movdqa XMMWORD[80+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1732	paddq xmm6,xmm6
				1733	pand xmm13,xmm12
				1734	pcmpgtd xmm14,xmm6
				1735	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1736	movdqu xmm11,XMMWORD[64+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1737	pxor xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1738	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1739	pxor xmm14,xmm14
				1740	movdqa xmm5,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1741	movdqa XMMWORD[96+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1742	paddq xmm6,xmm6
				1743	pand xmm13,xmm12
				1744	pcmpgtd xmm14,xmm6
				1745	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1746	movdqu xmm12,XMMWORD[80+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1747	pxor xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1748	movdqu xmm13,XMMWORD[96+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1749	pxor xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1750	movdqu xmm14,XMMWORD[112+r12]
				1751	lea r12,[128+r12]
				1752	movdqa XMMWORD[112+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1753	pxor xmm5,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1754	lea rax,[128+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1755	pxor xmm6,xmm14
				1756	mov r10d,edx
				1757
				1758	call _bsaes_encrypt8
				1759
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1760	pxor xmm15,XMMWORD[rsp]
				1761	pxor xmm0,XMMWORD[16+rsp]
				1762	movdqu XMMWORD[r13],xmm15
				1763	pxor xmm3,XMMWORD[32+rsp]
				1764	movdqu XMMWORD[16+r13],xmm0
				1765	pxor xmm5,XMMWORD[48+rsp]
				1766	movdqu XMMWORD[32+r13],xmm3
				1767	pxor xmm2,XMMWORD[64+rsp]
				1768	movdqu XMMWORD[48+r13],xmm5
				1769	pxor xmm6,XMMWORD[80+rsp]
				1770	movdqu XMMWORD[64+r13],xmm2
				1771	pxor xmm1,XMMWORD[96+rsp]
				1772	movdqu XMMWORD[80+r13],xmm6
				1773	pxor xmm4,XMMWORD[112+rsp]
				1774	movdqu XMMWORD[96+r13],xmm1
				1775	movdqu XMMWORD[112+r13],xmm4
				1776	lea r13,[128+r13]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1777
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1778	movdqa xmm6,XMMWORD[112+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1779	pxor xmm14,xmm14
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1780	movdqa xmm12,XMMWORD[$L$xts_magic]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1781	pcmpgtd xmm14,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1782	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1783	pxor xmm14,xmm14
				1784	paddq xmm6,xmm6
				1785	pand xmm13,xmm12
				1786	pcmpgtd xmm14,xmm6
				1787	pxor xmm6,xmm13
				1788
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1789	sub r14,0x80
				1790	jnc NEAR $L$xts_enc_loop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1791
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1792	$L$xts_enc_short:
				1793	add r14,0x80
				1794	jz NEAR $L$xts_enc_done
				1795	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1796	pxor xmm14,xmm14
				1797	movdqa xmm15,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1798	movdqa XMMWORD[rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1799	paddq xmm6,xmm6
				1800	pand xmm13,xmm12
				1801	pcmpgtd xmm14,xmm6
				1802	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1803	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1804	pxor xmm14,xmm14
				1805	movdqa xmm0,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1806	movdqa XMMWORD[16+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1807	paddq xmm6,xmm6
				1808	pand xmm13,xmm12
				1809	pcmpgtd xmm14,xmm6
				1810	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1811	movdqu xmm7,XMMWORD[r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1812	cmp r14,16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1813	je NEAR $L$xts_enc_1
				1814	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1815	pxor xmm14,xmm14
				1816	movdqa xmm1,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1817	movdqa XMMWORD[32+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1818	paddq xmm6,xmm6
				1819	pand xmm13,xmm12
				1820	pcmpgtd xmm14,xmm6
				1821	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1822	movdqu xmm8,XMMWORD[16+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1823	cmp r14,32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1824	je NEAR $L$xts_enc_2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1825	pxor xmm15,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1826	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1827	pxor xmm14,xmm14
				1828	movdqa xmm2,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1829	movdqa XMMWORD[48+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1830	paddq xmm6,xmm6
				1831	pand xmm13,xmm12
				1832	pcmpgtd xmm14,xmm6
				1833	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1834	movdqu xmm9,XMMWORD[32+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1835	cmp r14,48
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1836	je NEAR $L$xts_enc_3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1837	pxor xmm0,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1838	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1839	pxor xmm14,xmm14
				1840	movdqa xmm3,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1841	movdqa XMMWORD[64+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1842	paddq xmm6,xmm6
				1843	pand xmm13,xmm12
				1844	pcmpgtd xmm14,xmm6
				1845	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1846	movdqu xmm10,XMMWORD[48+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1847	cmp r14,64
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1848	je NEAR $L$xts_enc_4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1849	pxor xmm1,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1850	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1851	pxor xmm14,xmm14
				1852	movdqa xmm4,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1853	movdqa XMMWORD[80+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1854	paddq xmm6,xmm6
				1855	pand xmm13,xmm12
				1856	pcmpgtd xmm14,xmm6
				1857	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1858	movdqu xmm11,XMMWORD[64+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1859	cmp r14,80
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1860	je NEAR $L$xts_enc_5
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1861	pxor xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1862	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1863	pxor xmm14,xmm14
				1864	movdqa xmm5,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1865	movdqa XMMWORD[96+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1866	paddq xmm6,xmm6
				1867	pand xmm13,xmm12
				1868	pcmpgtd xmm14,xmm6
				1869	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1870	movdqu xmm12,XMMWORD[80+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1871	cmp r14,96
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1872	je NEAR $L$xts_enc_6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1873	pxor xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1874	movdqu xmm13,XMMWORD[96+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1875	pxor xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1876	movdqa XMMWORD[112+rsp],xmm6
				1877	lea r12,[112+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1878	pxor xmm5,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1879	lea rax,[128+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1880	mov r10d,edx
				1881
				1882	call _bsaes_encrypt8
				1883
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1884	pxor xmm15,XMMWORD[rsp]
				1885	pxor xmm0,XMMWORD[16+rsp]
				1886	movdqu XMMWORD[r13],xmm15
				1887	pxor xmm3,XMMWORD[32+rsp]
				1888	movdqu XMMWORD[16+r13],xmm0
				1889	pxor xmm5,XMMWORD[48+rsp]
				1890	movdqu XMMWORD[32+r13],xmm3
				1891	pxor xmm2,XMMWORD[64+rsp]
				1892	movdqu XMMWORD[48+r13],xmm5
				1893	pxor xmm6,XMMWORD[80+rsp]
				1894	movdqu XMMWORD[64+r13],xmm2
				1895	pxor xmm1,XMMWORD[96+rsp]
				1896	movdqu XMMWORD[80+r13],xmm6
				1897	movdqu XMMWORD[96+r13],xmm1
				1898	lea r13,[112+r13]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1899
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1900	movdqa xmm6,XMMWORD[112+rsp]
				1901	jmp NEAR $L$xts_enc_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1902	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1903	$L$xts_enc_6:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1904	pxor xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1905	lea r12,[96+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1906	pxor xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1907	lea rax,[128+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1908	mov r10d,edx
				1909
				1910	call _bsaes_encrypt8
				1911
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1912	pxor xmm15,XMMWORD[rsp]
				1913	pxor xmm0,XMMWORD[16+rsp]
				1914	movdqu XMMWORD[r13],xmm15
				1915	pxor xmm3,XMMWORD[32+rsp]
				1916	movdqu XMMWORD[16+r13],xmm0
				1917	pxor xmm5,XMMWORD[48+rsp]
				1918	movdqu XMMWORD[32+r13],xmm3
				1919	pxor xmm2,XMMWORD[64+rsp]
				1920	movdqu XMMWORD[48+r13],xmm5
				1921	pxor xmm6,XMMWORD[80+rsp]
				1922	movdqu XMMWORD[64+r13],xmm2
				1923	movdqu XMMWORD[80+r13],xmm6
				1924	lea r13,[96+r13]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1925
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1926	movdqa xmm6,XMMWORD[96+rsp]
				1927	jmp NEAR $L$xts_enc_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1928	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1929	$L$xts_enc_5:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1930	pxor xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1931	lea r12,[80+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1932	pxor xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1933	lea rax,[128+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1934	mov r10d,edx
				1935
				1936	call _bsaes_encrypt8
				1937
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1938	pxor xmm15,XMMWORD[rsp]
				1939	pxor xmm0,XMMWORD[16+rsp]
				1940	movdqu XMMWORD[r13],xmm15
				1941	pxor xmm3,XMMWORD[32+rsp]
				1942	movdqu XMMWORD[16+r13],xmm0
				1943	pxor xmm5,XMMWORD[48+rsp]
				1944	movdqu XMMWORD[32+r13],xmm3
				1945	pxor xmm2,XMMWORD[64+rsp]
				1946	movdqu XMMWORD[48+r13],xmm5
				1947	movdqu XMMWORD[64+r13],xmm2
				1948	lea r13,[80+r13]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1949
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1950	movdqa xmm6,XMMWORD[80+rsp]
				1951	jmp NEAR $L$xts_enc_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1952	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1953	$L$xts_enc_4:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1954	pxor xmm1,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1955	lea r12,[64+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1956	pxor xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1957	lea rax,[128+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1958	mov r10d,edx
				1959
				1960	call _bsaes_encrypt8
				1961
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1962	pxor xmm15,XMMWORD[rsp]
				1963	pxor xmm0,XMMWORD[16+rsp]
				1964	movdqu XMMWORD[r13],xmm15
				1965	pxor xmm3,XMMWORD[32+rsp]
				1966	movdqu XMMWORD[16+r13],xmm0
				1967	pxor xmm5,XMMWORD[48+rsp]
				1968	movdqu XMMWORD[32+r13],xmm3
				1969	movdqu XMMWORD[48+r13],xmm5
				1970	lea r13,[64+r13]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1971
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1972	movdqa xmm6,XMMWORD[64+rsp]
				1973	jmp NEAR $L$xts_enc_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1974	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1975	$L$xts_enc_3:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1976	pxor xmm0,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1977	lea r12,[48+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1978	pxor xmm1,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1979	lea rax,[128+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1980	mov r10d,edx
				1981
				1982	call _bsaes_encrypt8
				1983
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1984	pxor xmm15,XMMWORD[rsp]
				1985	pxor xmm0,XMMWORD[16+rsp]
				1986	movdqu XMMWORD[r13],xmm15
				1987	pxor xmm3,XMMWORD[32+rsp]
				1988	movdqu XMMWORD[16+r13],xmm0
				1989	movdqu XMMWORD[32+r13],xmm3
				1990	lea r13,[48+r13]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1991
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1992	movdqa xmm6,XMMWORD[48+rsp]
				1993	jmp NEAR $L$xts_enc_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1994	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1995	$L$xts_enc_2:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1996	pxor xmm15,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1997	lea r12,[32+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	1998	pxor xmm0,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	1999	lea rax,[128+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2000	mov r10d,edx
				2001
				2002	call _bsaes_encrypt8
				2003
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2004	pxor xmm15,XMMWORD[rsp]
				2005	pxor xmm0,XMMWORD[16+rsp]
				2006	movdqu XMMWORD[r13],xmm15
				2007	movdqu XMMWORD[16+r13],xmm0
				2008	lea r13,[32+r13]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2009
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2010	movdqa xmm6,XMMWORD[32+rsp]
				2011	jmp NEAR $L$xts_enc_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2012	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2013	$L$xts_enc_1:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2014	pxor xmm7,xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2015	lea r12,[16+r12]
				2016	movdqa XMMWORD[32+rbp],xmm7
				2017	lea rcx,[32+rbp]
				2018	lea rdx,[32+rbp]
				2019	lea r8,[r15]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2020	call asm_AES_encrypt
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2021	pxor xmm15,XMMWORD[32+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2022
				2023
				2024
				2025
				2026
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2027	movdqu XMMWORD[r13],xmm15
				2028	lea r13,[16+r13]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2029
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2030	movdqa xmm6,XMMWORD[16+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2031
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2032	$L$xts_enc_done:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2033	and ebx,15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2034	jz NEAR $L$xts_enc_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2035	mov rdx,r13
				2036
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2037	$L$xts_enc_steal:
				2038	movzx eax,BYTE[r12]
				2039	movzx ecx,BYTE[((-16))+rdx]
				2040	lea r12,[1+r12]
				2041	mov BYTE[((-16))+rdx],al
				2042	mov BYTE[rdx],cl
				2043	lea rdx,[1+rdx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2044	sub ebx,1
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2045	jnz NEAR $L$xts_enc_steal
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2046
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2047	movdqu xmm15,XMMWORD[((-16))+r13]
				2048	lea rcx,[32+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2049	pxor xmm15,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2050	lea rdx,[32+rbp]
				2051	movdqa XMMWORD[32+rbp],xmm15
				2052	lea r8,[r15]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2053	call asm_AES_encrypt
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2054	pxor xmm6,XMMWORD[32+rbp]
				2055	movdqu XMMWORD[(-16)+r13],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2056
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2057	$L$xts_enc_ret:
				2058	lea rax,[rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2059	pxor xmm0,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2060	$L$xts_enc_bzero:
				2061	movdqa XMMWORD[rax],xmm0
				2062	movdqa XMMWORD[16+rax],xmm0
				2063	lea rax,[32+rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2064	cmp rbp,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2065	ja NEAR $L$xts_enc_bzero
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2066
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2067	lea rax,[120+rbp]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2068
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2069	movaps xmm6,XMMWORD[64+rbp]
				2070	movaps xmm7,XMMWORD[80+rbp]
				2071	movaps xmm8,XMMWORD[96+rbp]
				2072	movaps xmm9,XMMWORD[112+rbp]
				2073	movaps xmm10,XMMWORD[128+rbp]
				2074	movaps xmm11,XMMWORD[144+rbp]
				2075	movaps xmm12,XMMWORD[160+rbp]
				2076	movaps xmm13,XMMWORD[176+rbp]
				2077	movaps xmm14,XMMWORD[192+rbp]
				2078	movaps xmm15,XMMWORD[208+rbp]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2079	lea rax,[160+rax]
				2080	$L$xts_enc_tail:
				2081	mov r15,QWORD[((-48))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2082
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2083	mov r14,QWORD[((-40))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2084
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2085	mov r13,QWORD[((-32))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2086
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2087	mov r12,QWORD[((-24))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2088
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2089	mov rbx,QWORD[((-16))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2090
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2091	mov rbp,QWORD[((-8))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2092
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2093	lea rsp,[rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2094
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2095	$L$xts_enc_epilogue:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2096	DB 0F3h,0C3h ;repret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2097
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2098
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2099
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2100	global bsaes_xts_decrypt
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2101
				2102	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2103	bsaes_xts_decrypt:
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2104
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2105	mov rax,rsp
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2106	$L$xts_dec_prologue:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2107	push rbp
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2108
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2109	push rbx
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2110
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2111	push r12
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2112
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2113	push r13
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2114
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2115	push r14
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2116
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2117	push r15
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2118
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2119	lea rsp,[((-72))+rsp]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2120
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2121	mov r10,QWORD[160+rsp]
				2122	mov r11,QWORD[168+rsp]
				2123	lea rsp,[((-160))+rsp]
				2124	movaps XMMWORD[64+rsp],xmm6
				2125	movaps XMMWORD[80+rsp],xmm7
				2126	movaps XMMWORD[96+rsp],xmm8
				2127	movaps XMMWORD[112+rsp],xmm9
				2128	movaps XMMWORD[128+rsp],xmm10
				2129	movaps XMMWORD[144+rsp],xmm11
				2130	movaps XMMWORD[160+rsp],xmm12
				2131	movaps XMMWORD[176+rsp],xmm13
				2132	movaps XMMWORD[192+rsp],xmm14
				2133	movaps XMMWORD[208+rsp],xmm15
				2134	$L$xts_dec_body:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2135	mov rbp,rsp
				2136	mov r12,rcx
				2137	mov r13,rdx
				2138	mov r14,r8
				2139	mov r15,r9
				2140
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2141	lea rcx,[r11]
				2142	lea rdx,[32+rbp]
				2143	lea r8,[r10]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2144	call asm_AES_encrypt
				2145
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2146	mov eax,DWORD[240+r15]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2147	mov rbx,r14
				2148
				2149	mov edx,eax
				2150	shl rax,7
				2151	sub rax,96
				2152	sub rsp,rax
				2153
				2154	mov rax,rsp
				2155	mov rcx,r15
				2156	mov r10d,edx
				2157	call _bsaes_key_convert
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2158	pxor xmm7,XMMWORD[rsp]
				2159	movdqa XMMWORD[rax],xmm6
				2160	movdqa XMMWORD[rsp],xmm7
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2161
				2162	xor eax,eax
				2163	and r14,-16
				2164	test ebx,15
				2165	setnz al
				2166	shl rax,4
				2167	sub r14,rax
				2168
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2169	sub rsp,0x80
				2170	movdqa xmm6,XMMWORD[32+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2171
				2172	pxor xmm14,xmm14
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2173	movdqa xmm12,XMMWORD[$L$xts_magic]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2174	pcmpgtd xmm14,xmm6
				2175
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2176	sub r14,0x80
				2177	jc NEAR $L$xts_dec_short
				2178	jmp NEAR $L$xts_dec_loop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2179
				2180	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2181	$L$xts_dec_loop:
				2182	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2183	pxor xmm14,xmm14
				2184	movdqa xmm15,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2185	movdqa XMMWORD[rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2186	paddq xmm6,xmm6
				2187	pand xmm13,xmm12
				2188	pcmpgtd xmm14,xmm6
				2189	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2190	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2191	pxor xmm14,xmm14
				2192	movdqa xmm0,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2193	movdqa XMMWORD[16+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2194	paddq xmm6,xmm6
				2195	pand xmm13,xmm12
				2196	pcmpgtd xmm14,xmm6
				2197	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2198	movdqu xmm7,XMMWORD[r12]
				2199	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2200	pxor xmm14,xmm14
				2201	movdqa xmm1,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2202	movdqa XMMWORD[32+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2203	paddq xmm6,xmm6
				2204	pand xmm13,xmm12
				2205	pcmpgtd xmm14,xmm6
				2206	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2207	movdqu xmm8,XMMWORD[16+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2208	pxor xmm15,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2209	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2210	pxor xmm14,xmm14
				2211	movdqa xmm2,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2212	movdqa XMMWORD[48+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2213	paddq xmm6,xmm6
				2214	pand xmm13,xmm12
				2215	pcmpgtd xmm14,xmm6
				2216	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2217	movdqu xmm9,XMMWORD[32+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2218	pxor xmm0,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2219	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2220	pxor xmm14,xmm14
				2221	movdqa xmm3,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2222	movdqa XMMWORD[64+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2223	paddq xmm6,xmm6
				2224	pand xmm13,xmm12
				2225	pcmpgtd xmm14,xmm6
				2226	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2227	movdqu xmm10,XMMWORD[48+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2228	pxor xmm1,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2229	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2230	pxor xmm14,xmm14
				2231	movdqa xmm4,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2232	movdqa XMMWORD[80+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2233	paddq xmm6,xmm6
				2234	pand xmm13,xmm12
				2235	pcmpgtd xmm14,xmm6
				2236	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2237	movdqu xmm11,XMMWORD[64+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2238	pxor xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2239	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2240	pxor xmm14,xmm14
				2241	movdqa xmm5,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2242	movdqa XMMWORD[96+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2243	paddq xmm6,xmm6
				2244	pand xmm13,xmm12
				2245	pcmpgtd xmm14,xmm6
				2246	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2247	movdqu xmm12,XMMWORD[80+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2248	pxor xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2249	movdqu xmm13,XMMWORD[96+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2250	pxor xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2251	movdqu xmm14,XMMWORD[112+r12]
				2252	lea r12,[128+r12]
				2253	movdqa XMMWORD[112+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2254	pxor xmm5,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2255	lea rax,[128+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2256	pxor xmm6,xmm14
				2257	mov r10d,edx
				2258
				2259	call _bsaes_decrypt8
				2260
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2261	pxor xmm15,XMMWORD[rsp]
				2262	pxor xmm0,XMMWORD[16+rsp]
				2263	movdqu XMMWORD[r13],xmm15
				2264	pxor xmm5,XMMWORD[32+rsp]
				2265	movdqu XMMWORD[16+r13],xmm0
				2266	pxor xmm3,XMMWORD[48+rsp]
				2267	movdqu XMMWORD[32+r13],xmm5
				2268	pxor xmm1,XMMWORD[64+rsp]
				2269	movdqu XMMWORD[48+r13],xmm3
				2270	pxor xmm6,XMMWORD[80+rsp]
				2271	movdqu XMMWORD[64+r13],xmm1
				2272	pxor xmm2,XMMWORD[96+rsp]
				2273	movdqu XMMWORD[80+r13],xmm6
				2274	pxor xmm4,XMMWORD[112+rsp]
				2275	movdqu XMMWORD[96+r13],xmm2
				2276	movdqu XMMWORD[112+r13],xmm4
				2277	lea r13,[128+r13]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2278
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2279	movdqa xmm6,XMMWORD[112+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2280	pxor xmm14,xmm14
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2281	movdqa xmm12,XMMWORD[$L$xts_magic]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2282	pcmpgtd xmm14,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2283	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2284	pxor xmm14,xmm14
				2285	paddq xmm6,xmm6
				2286	pand xmm13,xmm12
				2287	pcmpgtd xmm14,xmm6
				2288	pxor xmm6,xmm13
				2289
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2290	sub r14,0x80
				2291	jnc NEAR $L$xts_dec_loop
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2292
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2293	$L$xts_dec_short:
				2294	add r14,0x80
				2295	jz NEAR $L$xts_dec_done
				2296	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2297	pxor xmm14,xmm14
				2298	movdqa xmm15,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2299	movdqa XMMWORD[rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2300	paddq xmm6,xmm6
				2301	pand xmm13,xmm12
				2302	pcmpgtd xmm14,xmm6
				2303	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2304	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2305	pxor xmm14,xmm14
				2306	movdqa xmm0,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2307	movdqa XMMWORD[16+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2308	paddq xmm6,xmm6
				2309	pand xmm13,xmm12
				2310	pcmpgtd xmm14,xmm6
				2311	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2312	movdqu xmm7,XMMWORD[r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2313	cmp r14,16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2314	je NEAR $L$xts_dec_1
				2315	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2316	pxor xmm14,xmm14
				2317	movdqa xmm1,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2318	movdqa XMMWORD[32+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2319	paddq xmm6,xmm6
				2320	pand xmm13,xmm12
				2321	pcmpgtd xmm14,xmm6
				2322	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2323	movdqu xmm8,XMMWORD[16+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2324	cmp r14,32
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2325	je NEAR $L$xts_dec_2
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2326	pxor xmm15,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2327	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2328	pxor xmm14,xmm14
				2329	movdqa xmm2,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2330	movdqa XMMWORD[48+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2331	paddq xmm6,xmm6
				2332	pand xmm13,xmm12
				2333	pcmpgtd xmm14,xmm6
				2334	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2335	movdqu xmm9,XMMWORD[32+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2336	cmp r14,48
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2337	je NEAR $L$xts_dec_3
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2338	pxor xmm0,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2339	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2340	pxor xmm14,xmm14
				2341	movdqa xmm3,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2342	movdqa XMMWORD[64+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2343	paddq xmm6,xmm6
				2344	pand xmm13,xmm12
				2345	pcmpgtd xmm14,xmm6
				2346	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2347	movdqu xmm10,XMMWORD[48+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2348	cmp r14,64
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2349	je NEAR $L$xts_dec_4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2350	pxor xmm1,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2351	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2352	pxor xmm14,xmm14
				2353	movdqa xmm4,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2354	movdqa XMMWORD[80+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2355	paddq xmm6,xmm6
				2356	pand xmm13,xmm12
				2357	pcmpgtd xmm14,xmm6
				2358	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2359	movdqu xmm11,XMMWORD[64+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2360	cmp r14,80
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2361	je NEAR $L$xts_dec_5
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2362	pxor xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2363	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2364	pxor xmm14,xmm14
				2365	movdqa xmm5,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2366	movdqa XMMWORD[96+rsp],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2367	paddq xmm6,xmm6
				2368	pand xmm13,xmm12
				2369	pcmpgtd xmm14,xmm6
				2370	pxor xmm6,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2371	movdqu xmm12,XMMWORD[80+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2372	cmp r14,96
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2373	je NEAR $L$xts_dec_6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2374	pxor xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2375	movdqu xmm13,XMMWORD[96+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2376	pxor xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2377	movdqa XMMWORD[112+rsp],xmm6
				2378	lea r12,[112+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2379	pxor xmm5,xmm13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2380	lea rax,[128+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2381	mov r10d,edx
				2382
				2383	call _bsaes_decrypt8
				2384
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2385	pxor xmm15,XMMWORD[rsp]
				2386	pxor xmm0,XMMWORD[16+rsp]
				2387	movdqu XMMWORD[r13],xmm15
				2388	pxor xmm5,XMMWORD[32+rsp]
				2389	movdqu XMMWORD[16+r13],xmm0
				2390	pxor xmm3,XMMWORD[48+rsp]
				2391	movdqu XMMWORD[32+r13],xmm5
				2392	pxor xmm1,XMMWORD[64+rsp]
				2393	movdqu XMMWORD[48+r13],xmm3
				2394	pxor xmm6,XMMWORD[80+rsp]
				2395	movdqu XMMWORD[64+r13],xmm1
				2396	pxor xmm2,XMMWORD[96+rsp]
				2397	movdqu XMMWORD[80+r13],xmm6
				2398	movdqu XMMWORD[96+r13],xmm2
				2399	lea r13,[112+r13]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2400
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2401	movdqa xmm6,XMMWORD[112+rsp]
				2402	jmp NEAR $L$xts_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2403	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2404	$L$xts_dec_6:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2405	pxor xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2406	lea r12,[96+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2407	pxor xmm4,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2408	lea rax,[128+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2409	mov r10d,edx
				2410
				2411	call _bsaes_decrypt8
				2412
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2413	pxor xmm15,XMMWORD[rsp]
				2414	pxor xmm0,XMMWORD[16+rsp]
				2415	movdqu XMMWORD[r13],xmm15
				2416	pxor xmm5,XMMWORD[32+rsp]
				2417	movdqu XMMWORD[16+r13],xmm0
				2418	pxor xmm3,XMMWORD[48+rsp]
				2419	movdqu XMMWORD[32+r13],xmm5
				2420	pxor xmm1,XMMWORD[64+rsp]
				2421	movdqu XMMWORD[48+r13],xmm3
				2422	pxor xmm6,XMMWORD[80+rsp]
				2423	movdqu XMMWORD[64+r13],xmm1
				2424	movdqu XMMWORD[80+r13],xmm6
				2425	lea r13,[96+r13]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2426
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2427	movdqa xmm6,XMMWORD[96+rsp]
				2428	jmp NEAR $L$xts_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2429	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2430	$L$xts_dec_5:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2431	pxor xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2432	lea r12,[80+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2433	pxor xmm3,xmm11
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2434	lea rax,[128+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2435	mov r10d,edx
				2436
				2437	call _bsaes_decrypt8
				2438
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2439	pxor xmm15,XMMWORD[rsp]
				2440	pxor xmm0,XMMWORD[16+rsp]
				2441	movdqu XMMWORD[r13],xmm15
				2442	pxor xmm5,XMMWORD[32+rsp]
				2443	movdqu XMMWORD[16+r13],xmm0
				2444	pxor xmm3,XMMWORD[48+rsp]
				2445	movdqu XMMWORD[32+r13],xmm5
				2446	pxor xmm1,XMMWORD[64+rsp]
				2447	movdqu XMMWORD[48+r13],xmm3
				2448	movdqu XMMWORD[64+r13],xmm1
				2449	lea r13,[80+r13]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2450
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2451	movdqa xmm6,XMMWORD[80+rsp]
				2452	jmp NEAR $L$xts_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2453	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2454	$L$xts_dec_4:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2455	pxor xmm1,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2456	lea r12,[64+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2457	pxor xmm2,xmm10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2458	lea rax,[128+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2459	mov r10d,edx
				2460
				2461	call _bsaes_decrypt8
				2462
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2463	pxor xmm15,XMMWORD[rsp]
				2464	pxor xmm0,XMMWORD[16+rsp]
				2465	movdqu XMMWORD[r13],xmm15
				2466	pxor xmm5,XMMWORD[32+rsp]
				2467	movdqu XMMWORD[16+r13],xmm0
				2468	pxor xmm3,XMMWORD[48+rsp]
				2469	movdqu XMMWORD[32+r13],xmm5
				2470	movdqu XMMWORD[48+r13],xmm3
				2471	lea r13,[64+r13]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2472
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2473	movdqa xmm6,XMMWORD[64+rsp]
				2474	jmp NEAR $L$xts_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2475	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2476	$L$xts_dec_3:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2477	pxor xmm0,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2478	lea r12,[48+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2479	pxor xmm1,xmm9
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2480	lea rax,[128+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2481	mov r10d,edx
				2482
				2483	call _bsaes_decrypt8
				2484
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2485	pxor xmm15,XMMWORD[rsp]
				2486	pxor xmm0,XMMWORD[16+rsp]
				2487	movdqu XMMWORD[r13],xmm15
				2488	pxor xmm5,XMMWORD[32+rsp]
				2489	movdqu XMMWORD[16+r13],xmm0
				2490	movdqu XMMWORD[32+r13],xmm5
				2491	lea r13,[48+r13]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2492
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2493	movdqa xmm6,XMMWORD[48+rsp]
				2494	jmp NEAR $L$xts_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2495	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2496	$L$xts_dec_2:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2497	pxor xmm15,xmm7
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2498	lea r12,[32+r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2499	pxor xmm0,xmm8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2500	lea rax,[128+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2501	mov r10d,edx
				2502
				2503	call _bsaes_decrypt8
				2504
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2505	pxor xmm15,XMMWORD[rsp]
				2506	pxor xmm0,XMMWORD[16+rsp]
				2507	movdqu XMMWORD[r13],xmm15
				2508	movdqu XMMWORD[16+r13],xmm0
				2509	lea r13,[32+r13]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2510
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2511	movdqa xmm6,XMMWORD[32+rsp]
				2512	jmp NEAR $L$xts_dec_done
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2513	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2514	$L$xts_dec_1:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2515	pxor xmm7,xmm15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2516	lea r12,[16+r12]
				2517	movdqa XMMWORD[32+rbp],xmm7
				2518	lea rcx,[32+rbp]
				2519	lea rdx,[32+rbp]
				2520	lea r8,[r15]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2521	call asm_AES_decrypt
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2522	pxor xmm15,XMMWORD[32+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2523
				2524
				2525
				2526
				2527
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2528	movdqu XMMWORD[r13],xmm15
				2529	lea r13,[16+r13]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2530
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2531	movdqa xmm6,XMMWORD[16+rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2532
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2533	$L$xts_dec_done:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2534	and ebx,15
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2535	jz NEAR $L$xts_dec_ret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2536
				2537	pxor xmm14,xmm14
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2538	movdqa xmm12,XMMWORD[$L$xts_magic]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2539	pcmpgtd xmm14,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2540	pshufd xmm13,xmm14,0x13
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2541	movdqa xmm5,xmm6
				2542	paddq xmm6,xmm6
				2543	pand xmm13,xmm12
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2544	movdqu xmm15,XMMWORD[r12]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2545	pxor xmm6,xmm13
				2546
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2547	lea rcx,[32+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2548	pxor xmm15,xmm6
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2549	lea rdx,[32+rbp]
				2550	movdqa XMMWORD[32+rbp],xmm15
				2551	lea r8,[r15]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2552	call asm_AES_decrypt
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2553	pxor xmm6,XMMWORD[32+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2554	mov rdx,r13
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2555	movdqu XMMWORD[r13],xmm6
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2556
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2557	$L$xts_dec_steal:
				2558	movzx eax,BYTE[16+r12]
				2559	movzx ecx,BYTE[rdx]
				2560	lea r12,[1+r12]
				2561	mov BYTE[rdx],al
				2562	mov BYTE[16+rdx],cl
				2563	lea rdx,[1+rdx]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2564	sub ebx,1
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2565	jnz NEAR $L$xts_dec_steal
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2566
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2567	movdqu xmm15,XMMWORD[r13]
				2568	lea rcx,[32+rbp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2569	pxor xmm15,xmm5
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2570	lea rdx,[32+rbp]
				2571	movdqa XMMWORD[32+rbp],xmm15
				2572	lea r8,[r15]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2573	call asm_AES_decrypt
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2574	pxor xmm5,XMMWORD[32+rbp]
				2575	movdqu XMMWORD[r13],xmm5
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2576
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2577	$L$xts_dec_ret:
				2578	lea rax,[rsp]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2579	pxor xmm0,xmm0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2580	$L$xts_dec_bzero:
				2581	movdqa XMMWORD[rax],xmm0
				2582	movdqa XMMWORD[16+rax],xmm0
				2583	lea rax,[32+rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2584	cmp rbp,rax
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2585	ja NEAR $L$xts_dec_bzero
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2586
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2587	lea rax,[120+rbp]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2588
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2589	movaps xmm6,XMMWORD[64+rbp]
				2590	movaps xmm7,XMMWORD[80+rbp]
				2591	movaps xmm8,XMMWORD[96+rbp]
				2592	movaps xmm9,XMMWORD[112+rbp]
				2593	movaps xmm10,XMMWORD[128+rbp]
				2594	movaps xmm11,XMMWORD[144+rbp]
				2595	movaps xmm12,XMMWORD[160+rbp]
				2596	movaps xmm13,XMMWORD[176+rbp]
				2597	movaps xmm14,XMMWORD[192+rbp]
				2598	movaps xmm15,XMMWORD[208+rbp]
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2599	lea rax,[160+rax]
				2600	$L$xts_dec_tail:
				2601	mov r15,QWORD[((-48))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2602
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2603	mov r14,QWORD[((-40))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2604
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2605	mov r13,QWORD[((-32))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2606
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2607	mov r12,QWORD[((-24))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2608
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2609	mov rbx,QWORD[((-16))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2610
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2611	mov rbp,QWORD[((-8))+rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2612
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2613	lea rsp,[rax]
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2614
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2615	$L$xts_dec_epilogue:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2616	DB 0F3h,0C3h ;repret
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2617
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2618
Robert Sloan	ab8b888	2018-03-26 11:39:51 -0700	[diff] [blame^]	2619
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2620	ALIGN 64
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2621	_bsaes_const:
				2622	$L$M0ISR:
				2623	DQ 0x0a0e0206070b0f03,0x0004080c0d010509
				2624	$L$ISRM0:
				2625	DQ 0x01040b0e0205080f,0x0306090c00070a0d
				2626	$L$ISR:
				2627	DQ 0x0504070602010003,0x0f0e0d0c080b0a09
				2628	$L$BS0:
				2629	DQ 0x5555555555555555,0x5555555555555555
				2630	$L$BS1:
				2631	DQ 0x3333333333333333,0x3333333333333333
				2632	$L$BS2:
				2633	DQ 0x0f0f0f0f0f0f0f0f,0x0f0f0f0f0f0f0f0f
				2634	$L$SR:
				2635	DQ 0x0504070600030201,0x0f0e0d0c0a09080b
				2636	$L$SRM0:
				2637	DQ 0x0304090e00050a0f,0x01060b0c0207080d
				2638	$L$M0SR:
				2639	DQ 0x0a0e02060f03070b,0x0004080c05090d01
				2640	$L$SWPUP:
				2641	DQ 0x0706050403020100,0x0c0d0e0f0b0a0908
				2642	$L$SWPUPM0SR:
				2643	DQ 0x0a0d02060c03070b,0x0004080f05090e01
				2644	$L$ADD1:
				2645	DQ 0x0000000000000000,0x0000000100000000
				2646	$L$ADD2:
				2647	DQ 0x0000000000000000,0x0000000200000000
				2648	$L$ADD3:
				2649	DQ 0x0000000000000000,0x0000000300000000
				2650	$L$ADD4:
				2651	DQ 0x0000000000000000,0x0000000400000000
				2652	$L$ADD5:
				2653	DQ 0x0000000000000000,0x0000000500000000
				2654	$L$ADD6:
				2655	DQ 0x0000000000000000,0x0000000600000000
				2656	$L$ADD7:
				2657	DQ 0x0000000000000000,0x0000000700000000
				2658	$L$ADD8:
				2659	DQ 0x0000000000000000,0x0000000800000000
				2660	$L$xts_magic:
				2661	DD 0x87,0,1,0
				2662	$L$masks:
				2663	DQ 0x0101010101010101,0x0101010101010101
				2664	DQ 0x0202020202020202,0x0202020202020202
				2665	DQ 0x0404040404040404,0x0404040404040404
				2666	DQ 0x0808080808080808,0x0808080808080808
				2667	$L$M0:
				2668	DQ 0x02060a0e03070b0f,0x0004080c0105090d
				2669	$L$63:
				2670	DQ 0x6363636363636363,0x6363636363636363
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2671	DB 66,105,116,45,115,108,105,99,101,100,32,65,69,83,32,102
				2672	DB 111,114,32,120,56,54,95,54,52,47,83,83,83,69,51,44
				2673	DB 32,69,109,105,108,105,97,32,75,195,164,115,112,101,114,44
				2674	DB 32,80,101,116,101,114,32,83,99,104,119,97,98,101,44,32
				2675	DB 65,110,100,121,32,80,111,108,121,97,107,111,118,0
				2676	ALIGN 64
				2677
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2678	EXTERN __imp_RtlVirtualUnwind
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2679
				2680	ALIGN 16
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2681	se_handler:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2682	push rsi
				2683	push rdi
				2684	push rbx
				2685	push rbp
				2686	push r12
				2687	push r13
				2688	push r14
				2689	push r15
				2690	pushfq
				2691	sub rsp,64
				2692
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2693	mov rax,QWORD[120+r8]
				2694	mov rbx,QWORD[248+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2695
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2696	mov rsi,QWORD[8+r9]
				2697	mov r11,QWORD[56+r9]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2698
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2699	mov r10d,DWORD[r11]
				2700	lea r10,[r10*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2701	cmp rbx,r10
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2702	jbe NEAR $L$in_prologue
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2703
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2704	mov r10d,DWORD[4+r11]
				2705	lea r10,[r10*1+rsi]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2706	cmp rbx,r10
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2707	jae NEAR $L$in_prologue
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2708
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2709	mov r10d,DWORD[8+r11]
				2710	lea r10,[r10*1+rsi]
				2711	cmp rbx,r10
				2712	jae NEAR $L$in_tail
				2713
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2714	mov rax,QWORD[160+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2715
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2716	lea rsi,[64+rax]
				2717	lea rdi,[512+r8]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2718	mov ecx,20
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2719	DD 0xa548f3fc
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2720	lea rax,[((160+120))+rax]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2721
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2722	$L$in_tail:
				2723	mov rbp,QWORD[((-48))+rax]
				2724	mov rbx,QWORD[((-40))+rax]
				2725	mov r12,QWORD[((-32))+rax]
				2726	mov r13,QWORD[((-24))+rax]
				2727	mov r14,QWORD[((-16))+rax]
				2728	mov r15,QWORD[((-8))+rax]
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2729	mov QWORD[144+r8],rbx
				2730	mov QWORD[160+r8],rbp
				2731	mov QWORD[216+r8],r12
				2732	mov QWORD[224+r8],r13
				2733	mov QWORD[232+r8],r14
				2734	mov QWORD[240+r8],r15
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2735
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2736	$L$in_prologue:
				2737	mov QWORD[152+r8],rax
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2738
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2739	mov rdi,QWORD[40+r9]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2740	mov rsi,r8
				2741	mov ecx,154
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2742	DD 0xa548f3fc
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2743
				2744	mov rsi,r9
				2745	xor rcx,rcx
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2746	mov rdx,QWORD[8+rsi]
				2747	mov r8,QWORD[rsi]
				2748	mov r9,QWORD[16+rsi]
				2749	mov r10,QWORD[40+rsi]
				2750	lea r11,[56+rsi]
				2751	lea r12,[24+rsi]
				2752	mov QWORD[32+rsp],r10
				2753	mov QWORD[40+rsp],r11
				2754	mov QWORD[48+rsp],r12
				2755	mov QWORD[56+rsp],rcx
				2756	call QWORD[__imp_RtlVirtualUnwind]
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2757
				2758	mov eax,1
				2759	add rsp,64
				2760	popfq
				2761	pop r15
				2762	pop r14
				2763	pop r13
				2764	pop r12
				2765	pop rbp
				2766	pop rbx
				2767	pop rdi
				2768	pop rsi
				2769	DB 0F3h,0C3h ;repret
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2770
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2771
				2772	section .pdata rdata align=4
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2773	ALIGN 4
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2774	DD $L$cbc_dec_prologue wrt ..imagebase
				2775	DD $L$cbc_dec_epilogue wrt ..imagebase
				2776	DD $L$cbc_dec_info wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2777
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2778	DD $L$ctr_enc_prologue wrt ..imagebase
				2779	DD $L$ctr_enc_epilogue wrt ..imagebase
				2780	DD $L$ctr_enc_info wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2781
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2782	DD $L$xts_enc_prologue wrt ..imagebase
				2783	DD $L$xts_enc_epilogue wrt ..imagebase
				2784	DD $L$xts_enc_info wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2785
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2786	DD $L$xts_dec_prologue wrt ..imagebase
				2787	DD $L$xts_dec_epilogue wrt ..imagebase
				2788	DD $L$xts_dec_info wrt ..imagebase
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2789
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2790	section .xdata rdata align=8
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2791	ALIGN 8
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2792	$L$cbc_dec_info:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2793	DB 9,0,0,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2794	DD se_handler wrt ..imagebase
				2795	DD $L$cbc_dec_body wrt ..imagebase,$L$cbc_dec_epilogue wrt ..imagebase
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2796	DD $L$cbc_dec_tail wrt ..imagebase
				2797	DD 0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2798	$L$ctr_enc_info:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2799	DB 9,0,0,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2800	DD se_handler wrt ..imagebase
				2801	DD $L$ctr_enc_body wrt ..imagebase,$L$ctr_enc_epilogue wrt ..imagebase
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2802	DD $L$ctr_enc_tail wrt ..imagebase
				2803	DD 0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2804	$L$xts_enc_info:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2805	DB 9,0,0,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2806	DD se_handler wrt ..imagebase
				2807	DD $L$xts_enc_body wrt ..imagebase,$L$xts_enc_epilogue wrt ..imagebase
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2808	DD $L$xts_enc_tail wrt ..imagebase
				2809	DD 0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2810	$L$xts_dec_info:
Adam Langley	d9e397b	2015-01-22 14:27:53 -0800	[diff] [blame]	2811	DB 9,0,0,0
Adam Langley	e9ada86	2015-05-11 17:20:37 -0700	[diff] [blame]	2812	DD se_handler wrt ..imagebase
				2813	DD $L$xts_dec_body wrt ..imagebase,$L$xts_dec_epilogue wrt ..imagebase
Robert Sloan	a94fe05	2017-02-21 08:49:28 -0800	[diff] [blame]	2814	DD $L$xts_dec_tail wrt ..imagebase
				2815	DD 0