blob: 6e93004f2181ccb3f265994cf9806db528641796 [file] [log] [blame]
Herbert Xu5cde0af2006-08-22 00:07:53 +10001/*
2 * Block chaining cipher operations.
3 *
4 * Generic encrypt/decrypt wrapper for ciphers, handles operations across
5 * multiple page boundaries by using temporary blocks. In user context,
6 * the kernel is given a chance to schedule us once per page.
7 *
8 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
9 *
10 * This program is free software; you can redistribute it and/or modify it
11 * under the terms of the GNU General Public License as published by the Free
12 * Software Foundation; either version 2 of the License, or (at your option)
13 * any later version.
14 *
15 */
16
17#include <linux/crypto.h>
18#include <linux/errno.h>
19#include <linux/kernel.h>
Herbert Xu5cde0af2006-08-22 00:07:53 +100020#include <linux/module.h>
21#include <linux/scatterlist.h>
22#include <linux/seq_file.h>
23#include <linux/slab.h>
24#include <linux/string.h>
25
26#include "internal.h"
27#include "scatterwalk.h"
28
29enum {
30 BLKCIPHER_WALK_PHYS = 1 << 0,
31 BLKCIPHER_WALK_SLOW = 1 << 1,
32 BLKCIPHER_WALK_COPY = 1 << 2,
33 BLKCIPHER_WALK_DIFF = 1 << 3,
34};
35
36static int blkcipher_walk_next(struct blkcipher_desc *desc,
37 struct blkcipher_walk *walk);
38static int blkcipher_walk_first(struct blkcipher_desc *desc,
39 struct blkcipher_walk *walk);
40
41static inline void blkcipher_map_src(struct blkcipher_walk *walk)
42{
43 walk->src.virt.addr = scatterwalk_map(&walk->in, 0);
44}
45
46static inline void blkcipher_map_dst(struct blkcipher_walk *walk)
47{
48 walk->dst.virt.addr = scatterwalk_map(&walk->out, 1);
49}
50
51static inline void blkcipher_unmap_src(struct blkcipher_walk *walk)
52{
53 scatterwalk_unmap(walk->src.virt.addr, 0);
54}
55
56static inline void blkcipher_unmap_dst(struct blkcipher_walk *walk)
57{
58 scatterwalk_unmap(walk->dst.virt.addr, 1);
59}
60
61static inline u8 *blkcipher_get_spot(u8 *start, unsigned int len)
62{
63 if (offset_in_page(start + len) < len)
64 return (u8 *)((unsigned long)(start + len) & PAGE_MASK);
65 return start;
66}
67
68static inline unsigned int blkcipher_done_slow(struct crypto_blkcipher *tfm,
69 struct blkcipher_walk *walk,
70 unsigned int bsize)
71{
72 u8 *addr;
73 unsigned int alignmask = crypto_blkcipher_alignmask(tfm);
74
75 addr = (u8 *)ALIGN((unsigned long)walk->buffer, alignmask + 1);
76 addr = blkcipher_get_spot(addr, bsize);
77 scatterwalk_copychunks(addr, &walk->out, bsize, 1);
78 return bsize;
79}
80
81static inline unsigned int blkcipher_done_fast(struct blkcipher_walk *walk,
82 unsigned int n)
83{
84 n = walk->nbytes - n;
85
86 if (walk->flags & BLKCIPHER_WALK_COPY) {
87 blkcipher_map_dst(walk);
88 memcpy(walk->dst.virt.addr, walk->page, n);
89 blkcipher_unmap_dst(walk);
90 } else if (!(walk->flags & BLKCIPHER_WALK_PHYS)) {
91 blkcipher_unmap_src(walk);
92 if (walk->flags & BLKCIPHER_WALK_DIFF)
93 blkcipher_unmap_dst(walk);
94 }
95
96 scatterwalk_advance(&walk->in, n);
97 scatterwalk_advance(&walk->out, n);
98
99 return n;
100}
101
102int blkcipher_walk_done(struct blkcipher_desc *desc,
103 struct blkcipher_walk *walk, int err)
104{
105 struct crypto_blkcipher *tfm = desc->tfm;
106 unsigned int nbytes = 0;
107
108 if (likely(err >= 0)) {
109 unsigned int bsize = crypto_blkcipher_blocksize(tfm);
110 unsigned int n;
111
112 if (likely(!(walk->flags & BLKCIPHER_WALK_SLOW)))
113 n = blkcipher_done_fast(walk, err);
114 else
115 n = blkcipher_done_slow(tfm, walk, bsize);
116
117 nbytes = walk->total - n;
118 err = 0;
119 }
120
121 scatterwalk_done(&walk->in, 0, nbytes);
122 scatterwalk_done(&walk->out, 1, nbytes);
123
124 walk->total = nbytes;
125 walk->nbytes = nbytes;
126
127 if (nbytes) {
128 crypto_yield(desc->flags);
129 return blkcipher_walk_next(desc, walk);
130 }
131
132 if (walk->iv != desc->info)
133 memcpy(desc->info, walk->iv, crypto_blkcipher_ivsize(tfm));
134 if (walk->buffer != walk->page)
135 kfree(walk->buffer);
136 if (walk->page)
137 free_page((unsigned long)walk->page);
138
139 return err;
140}
141EXPORT_SYMBOL_GPL(blkcipher_walk_done);
142
143static inline int blkcipher_next_slow(struct blkcipher_desc *desc,
144 struct blkcipher_walk *walk,
145 unsigned int bsize,
146 unsigned int alignmask)
147{
148 unsigned int n;
149
150 if (walk->buffer)
151 goto ok;
152
153 walk->buffer = walk->page;
154 if (walk->buffer)
155 goto ok;
156
157 n = bsize * 2 + (alignmask & ~(crypto_tfm_ctx_alignment() - 1));
158 walk->buffer = kmalloc(n, GFP_ATOMIC);
159 if (!walk->buffer)
160 return blkcipher_walk_done(desc, walk, -ENOMEM);
161
162ok:
163 walk->dst.virt.addr = (u8 *)ALIGN((unsigned long)walk->buffer,
164 alignmask + 1);
165 walk->dst.virt.addr = blkcipher_get_spot(walk->dst.virt.addr, bsize);
166 walk->src.virt.addr = blkcipher_get_spot(walk->dst.virt.addr + bsize,
167 bsize);
168
169 scatterwalk_copychunks(walk->src.virt.addr, &walk->in, bsize, 0);
170
171 walk->nbytes = bsize;
172 walk->flags |= BLKCIPHER_WALK_SLOW;
173
174 return 0;
175}
176
177static inline int blkcipher_next_copy(struct blkcipher_walk *walk)
178{
179 u8 *tmp = walk->page;
180
181 blkcipher_map_src(walk);
182 memcpy(tmp, walk->src.virt.addr, walk->nbytes);
183 blkcipher_unmap_src(walk);
184
185 walk->src.virt.addr = tmp;
186 walk->dst.virt.addr = tmp;
187
188 return 0;
189}
190
191static inline int blkcipher_next_fast(struct blkcipher_desc *desc,
192 struct blkcipher_walk *walk)
193{
194 unsigned long diff;
195
196 walk->src.phys.page = scatterwalk_page(&walk->in);
197 walk->src.phys.offset = offset_in_page(walk->in.offset);
198 walk->dst.phys.page = scatterwalk_page(&walk->out);
199 walk->dst.phys.offset = offset_in_page(walk->out.offset);
200
201 if (walk->flags & BLKCIPHER_WALK_PHYS)
202 return 0;
203
204 diff = walk->src.phys.offset - walk->dst.phys.offset;
205 diff |= walk->src.virt.page - walk->dst.virt.page;
206
207 blkcipher_map_src(walk);
208 walk->dst.virt.addr = walk->src.virt.addr;
209
210 if (diff) {
211 walk->flags |= BLKCIPHER_WALK_DIFF;
212 blkcipher_map_dst(walk);
213 }
214
215 return 0;
216}
217
218static int blkcipher_walk_next(struct blkcipher_desc *desc,
219 struct blkcipher_walk *walk)
220{
221 struct crypto_blkcipher *tfm = desc->tfm;
222 unsigned int alignmask = crypto_blkcipher_alignmask(tfm);
223 unsigned int bsize = crypto_blkcipher_blocksize(tfm);
224 unsigned int n;
225 int err;
226
227 n = walk->total;
228 if (unlikely(n < bsize)) {
229 desc->flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
230 return blkcipher_walk_done(desc, walk, -EINVAL);
231 }
232
233 walk->flags &= ~(BLKCIPHER_WALK_SLOW | BLKCIPHER_WALK_COPY |
234 BLKCIPHER_WALK_DIFF);
235 if (!scatterwalk_aligned(&walk->in, alignmask) ||
236 !scatterwalk_aligned(&walk->out, alignmask)) {
237 walk->flags |= BLKCIPHER_WALK_COPY;
238 if (!walk->page) {
239 walk->page = (void *)__get_free_page(GFP_ATOMIC);
240 if (!walk->page)
241 n = 0;
242 }
243 }
244
245 n = scatterwalk_clamp(&walk->in, n);
246 n = scatterwalk_clamp(&walk->out, n);
247
248 if (unlikely(n < bsize)) {
249 err = blkcipher_next_slow(desc, walk, bsize, alignmask);
250 goto set_phys_lowmem;
251 }
252
253 walk->nbytes = n;
254 if (walk->flags & BLKCIPHER_WALK_COPY) {
255 err = blkcipher_next_copy(walk);
256 goto set_phys_lowmem;
257 }
258
259 return blkcipher_next_fast(desc, walk);
260
261set_phys_lowmem:
262 if (walk->flags & BLKCIPHER_WALK_PHYS) {
263 walk->src.phys.page = virt_to_page(walk->src.virt.addr);
264 walk->dst.phys.page = virt_to_page(walk->dst.virt.addr);
265 walk->src.phys.offset &= PAGE_SIZE - 1;
266 walk->dst.phys.offset &= PAGE_SIZE - 1;
267 }
268 return err;
269}
270
271static inline int blkcipher_copy_iv(struct blkcipher_walk *walk,
272 struct crypto_blkcipher *tfm,
273 unsigned int alignmask)
274{
275 unsigned bs = crypto_blkcipher_blocksize(tfm);
276 unsigned int ivsize = crypto_blkcipher_ivsize(tfm);
277 unsigned int size = bs * 2 + ivsize + max(bs, ivsize) - (alignmask + 1);
278 u8 *iv;
279
280 size += alignmask & ~(crypto_tfm_ctx_alignment() - 1);
281 walk->buffer = kmalloc(size, GFP_ATOMIC);
282 if (!walk->buffer)
283 return -ENOMEM;
284
285 iv = (u8 *)ALIGN((unsigned long)walk->buffer, alignmask + 1);
286 iv = blkcipher_get_spot(iv, bs) + bs;
287 iv = blkcipher_get_spot(iv, bs) + bs;
288 iv = blkcipher_get_spot(iv, ivsize);
289
290 walk->iv = memcpy(iv, walk->iv, ivsize);
291 return 0;
292}
293
294int blkcipher_walk_virt(struct blkcipher_desc *desc,
295 struct blkcipher_walk *walk)
296{
297 walk->flags &= ~BLKCIPHER_WALK_PHYS;
298 return blkcipher_walk_first(desc, walk);
299}
300EXPORT_SYMBOL_GPL(blkcipher_walk_virt);
301
302int blkcipher_walk_phys(struct blkcipher_desc *desc,
303 struct blkcipher_walk *walk)
304{
305 walk->flags |= BLKCIPHER_WALK_PHYS;
306 return blkcipher_walk_first(desc, walk);
307}
308EXPORT_SYMBOL_GPL(blkcipher_walk_phys);
309
310static int blkcipher_walk_first(struct blkcipher_desc *desc,
311 struct blkcipher_walk *walk)
312{
313 struct crypto_blkcipher *tfm = desc->tfm;
314 unsigned int alignmask = crypto_blkcipher_alignmask(tfm);
315
316 walk->nbytes = walk->total;
317 if (unlikely(!walk->total))
318 return 0;
319
320 walk->buffer = NULL;
321 walk->iv = desc->info;
322 if (unlikely(((unsigned long)walk->iv & alignmask))) {
323 int err = blkcipher_copy_iv(walk, tfm, alignmask);
324 if (err)
325 return err;
326 }
327
328 scatterwalk_start(&walk->in, walk->in.sg);
329 scatterwalk_start(&walk->out, walk->out.sg);
330 walk->page = NULL;
331
332 return blkcipher_walk_next(desc, walk);
333}
334
335static int setkey(struct crypto_tfm *tfm, const u8 *key,
336 unsigned int keylen)
337{
338 struct blkcipher_alg *cipher = &tfm->__crt_alg->cra_blkcipher;
339
340 if (keylen < cipher->min_keysize || keylen > cipher->max_keysize) {
341 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
342 return -EINVAL;
343 }
344
345 return cipher->setkey(tfm, key, keylen);
346}
347
348static unsigned int crypto_blkcipher_ctxsize(struct crypto_alg *alg)
349{
350 struct blkcipher_alg *cipher = &alg->cra_blkcipher;
351 unsigned int len = alg->cra_ctxsize;
352
353 if (cipher->ivsize) {
354 len = ALIGN(len, (unsigned long)alg->cra_alignmask + 1);
355 len += cipher->ivsize;
356 }
357
358 return len;
359}
360
361static int crypto_init_blkcipher_ops(struct crypto_tfm *tfm)
362{
363 struct blkcipher_tfm *crt = &tfm->crt_blkcipher;
364 struct blkcipher_alg *alg = &tfm->__crt_alg->cra_blkcipher;
365 unsigned long align = crypto_tfm_alg_alignmask(tfm) + 1;
366 unsigned long addr;
367
368 if (alg->ivsize > PAGE_SIZE / 8)
369 return -EINVAL;
370
371 crt->setkey = setkey;
372 crt->encrypt = alg->encrypt;
373 crt->decrypt = alg->decrypt;
374
375 addr = (unsigned long)crypto_tfm_ctx(tfm);
376 addr = ALIGN(addr, align);
377 addr += ALIGN(tfm->__crt_alg->cra_ctxsize, align);
378 crt->iv = (void *)addr;
379
380 return 0;
381}
382
383static void crypto_blkcipher_show(struct seq_file *m, struct crypto_alg *alg)
384 __attribute_used__;
385static void crypto_blkcipher_show(struct seq_file *m, struct crypto_alg *alg)
386{
387 seq_printf(m, "type : blkcipher\n");
388 seq_printf(m, "blocksize : %u\n", alg->cra_blocksize);
389 seq_printf(m, "min keysize : %u\n", alg->cra_blkcipher.min_keysize);
390 seq_printf(m, "max keysize : %u\n", alg->cra_blkcipher.max_keysize);
391 seq_printf(m, "ivsize : %u\n", alg->cra_blkcipher.ivsize);
392}
393
394const struct crypto_type crypto_blkcipher_type = {
395 .ctxsize = crypto_blkcipher_ctxsize,
396 .init = crypto_init_blkcipher_ops,
397#ifdef CONFIG_PROC_FS
398 .show = crypto_blkcipher_show,
399#endif
400};
401EXPORT_SYMBOL_GPL(crypto_blkcipher_type);
402
403MODULE_LICENSE("GPL");
404MODULE_DESCRIPTION("Generic block chaining cipher type");