Gitiles
Code Review Sign In
gerrit-public.fairphone.software / kernel / msm-4.9 / 564ec0ec05ac6ee409bde81f7ef27a3dadbf3a6a / . / arch / x86 / crypto / ghash-clmulni-intel_asm.S
blob: 1528dc4886cf88264e6eb2b764e27ac1029ebbfe [file] [log] [blame]
Huang Ying0e1227d2009-10-19 11:53:06 +0900[diff] [blame]1/*
2 * Accelerated GHASH implementation with Intel PCLMULQDQ-NI
3 * instructions. This file contains accelerated part of ghash
4 * implementation. More information about PCLMULQDQ can be found at:
5 *
6 * http://software.intel.com/en-us/articles/carry-less-multiplication-and-its-usage-for-computing-the-gcm-mode/
7 *
8 * Copyright (c) 2009 Intel Corp.
9 * Author: Huang Ying <ying.huang@intel.com>
10 * Vinodh Gopal
11 * Erdinc Ozturk
12 * Deniz Karakoyunlu
13 *
14 * This program is free software; you can redistribute it and/or modify it
15 * under the terms of the GNU General Public License version 2 as published
16 * by the Free Software Foundation.
17 */
18
19#include <linux/linkage.h>
Huang Ying564ec0e2009-11-23 19:55:22 +0800[diff] [blame^]20#include <asm/inst.h>
Huang Ying0e1227d2009-10-19 11:53:06 +0900[diff] [blame]21
22.align 16
23.Lbswap_mask:
24 .octa 0x000102030405060708090a0b0c0d0e0f
25.Lpoly:
26 .octa 0xc2000000000000000000000000000001
27.Ltwo_one:
28 .octa 0x00000001000000000000000000000001
29
30#define DATA %xmm0
31#define SHASH %xmm1
32#define T1 %xmm2
33#define T2 %xmm3
34#define T3 %xmm4
35#define BSWAP %xmm5
36#define IN1 %xmm6
37
38.text
39
40/*
41 * __clmul_gf128mul_ble: internal ABI
42 * input:
43 * DATA: operand1
44 * SHASH: operand2, hash_key << 1 mod poly
45 * output:
46 * DATA: operand1 * operand2 mod poly
47 * changed:
48 * T1
49 * T2
50 * T3
51 */
52__clmul_gf128mul_ble:
53 movaps DATA, T1
54 pshufd $0b01001110, DATA, T2
55 pshufd $0b01001110, SHASH, T3
56 pxor DATA, T2
57 pxor SHASH, T3
58
Huang Ying564ec0e2009-11-23 19:55:22 +0800[diff] [blame^]59 PCLMULQDQ 0x00 SHASH DATA # DATA = a0 * b0
60 PCLMULQDQ 0x11 SHASH T1 # T1 = a1 * b1
61 PCLMULQDQ 0x00 T3 T2 # T2 = (a1 + a0) * (b1 + b0)
Huang Ying0e1227d2009-10-19 11:53:06 +0900[diff] [blame]62 pxor DATA, T2
63 pxor T1, T2 # T2 = a0 * b1 + a1 * b0
64
65 movaps T2, T3
66 pslldq $8, T3
67 psrldq $8, T2
68 pxor T3, DATA
69 pxor T2, T1 # <T1:DATA> is result of
70 # carry-less multiplication
71
72 # first phase of the reduction
73 movaps DATA, T3
74 psllq $1, T3
75 pxor DATA, T3
76 psllq $5, T3
77 pxor DATA, T3
78 psllq $57, T3
79 movaps T3, T2
80 pslldq $8, T2
81 psrldq $8, T3
82 pxor T2, DATA
83 pxor T3, T1
84
85 # second phase of the reduction
86 movaps DATA, T2
87 psrlq $5, T2
88 pxor DATA, T2
89 psrlq $1, T2
90 pxor DATA, T2
91 psrlq $1, T2
92 pxor T2, T1
93 pxor T1, DATA
94 ret
95
96/* void clmul_ghash_mul(char *dst, const be128 *shash) */
97ENTRY(clmul_ghash_mul)
98 movups (%rdi), DATA
99 movups (%rsi), SHASH
100 movaps .Lbswap_mask, BSWAP
Huang Ying564ec0e2009-11-23 19:55:22 +0800[diff] [blame^]101 PSHUFB_XMM BSWAP DATA
Huang Ying0e1227d2009-10-19 11:53:06 +0900[diff] [blame]102 call __clmul_gf128mul_ble
Huang Ying564ec0e2009-11-23 19:55:22 +0800[diff] [blame^]103 PSHUFB_XMM BSWAP DATA
Huang Ying0e1227d2009-10-19 11:53:06 +0900[diff] [blame]104 movups DATA, (%rdi)
105 ret
106
107/*
108 * void clmul_ghash_update(char *dst, const char *src, unsigned int srclen,
109 * const be128 *shash);
110 */
111ENTRY(clmul_ghash_update)
112 cmp $16, %rdx
113 jb .Lupdate_just_ret # check length
114 movaps .Lbswap_mask, BSWAP
115 movups (%rdi), DATA
116 movups (%rcx), SHASH
Huang Ying564ec0e2009-11-23 19:55:22 +0800[diff] [blame^]117 PSHUFB_XMM BSWAP DATA
Huang Ying0e1227d2009-10-19 11:53:06 +0900[diff] [blame]118.align 4
119.Lupdate_loop:
120 movups (%rsi), IN1
Huang Ying564ec0e2009-11-23 19:55:22 +0800[diff] [blame^]121 PSHUFB_XMM BSWAP IN1
Huang Ying0e1227d2009-10-19 11:53:06 +0900[diff] [blame]122 pxor IN1, DATA
123 call __clmul_gf128mul_ble
124 sub $16, %rdx
125 add $16, %rsi
126 cmp $16, %rdx
127 jge .Lupdate_loop
Huang Ying564ec0e2009-11-23 19:55:22 +0800[diff] [blame^]128 PSHUFB_XMM BSWAP DATA
Huang Ying0e1227d2009-10-19 11:53:06 +0900[diff] [blame]129 movups DATA, (%rdi)
130.Lupdate_just_ret:
131 ret
132
133/*
134 * void clmul_ghash_setkey(be128 *shash, const u8 *key);
135 *
136 * Calculate hash_key << 1 mod poly
137 */
138ENTRY(clmul_ghash_setkey)
139 movaps .Lbswap_mask, BSWAP
140 movups (%rsi), %xmm0
Huang Ying564ec0e2009-11-23 19:55:22 +0800[diff] [blame^]141 PSHUFB_XMM BSWAP %xmm0
Huang Ying0e1227d2009-10-19 11:53:06 +0900[diff] [blame]142 movaps %xmm0, %xmm1
143 psllq $1, %xmm0
144 psrlq $63, %xmm1
145 movaps %xmm1, %xmm2
146 pslldq $8, %xmm1
147 psrldq $8, %xmm2
148 por %xmm1, %xmm0
149 # reduction
150 pshufd $0b00100100, %xmm2, %xmm1
151 pcmpeqd .Ltwo_one, %xmm1
152 pand .Lpoly, %xmm1
153 pxor %xmm1, %xmm0
154 movups %xmm0, (%rdi)
155 ret