Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2014 Sergey Senozhatsky. |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or |
| 5 | * modify it under the terms of the GNU General Public License |
| 6 | * as published by the Free Software Foundation; either version |
| 7 | * 2 of the License, or (at your option) any later version. |
| 8 | */ |
| 9 | |
| 10 | #include <linux/kernel.h> |
| 11 | #include <linux/string.h> |
Sergey Senozhatsky | fcfa8d9 | 2014-04-07 15:38:20 -0700 | [diff] [blame] | 12 | #include <linux/err.h> |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 13 | #include <linux/slab.h> |
| 14 | #include <linux/wait.h> |
| 15 | #include <linux/sched.h> |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 16 | #include <linux/cpu.h> |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 17 | #include <linux/crypto.h> |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 18 | |
| 19 | #include "zcomp.h" |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 20 | |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 21 | static const char * const backends[] = { |
| 22 | "lzo", |
Sergey Senozhatsky | ce1ed9f | 2016-07-26 15:22:54 -0700 | [diff] [blame] | 23 | #if IS_ENABLED(CONFIG_CRYPTO_LZ4) |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 24 | "lz4", |
Sergey Senozhatsky | 6e76668 | 2014-04-07 15:38:18 -0700 | [diff] [blame] | 25 | #endif |
Sergey Senozhatsky | eb9f56d | 2016-07-26 15:22:56 -0700 | [diff] [blame] | 26 | #if IS_ENABLED(CONFIG_CRYPTO_DEFLATE) |
| 27 | "deflate", |
| 28 | #endif |
| 29 | #if IS_ENABLED(CONFIG_CRYPTO_LZ4HC) |
| 30 | "lz4hc", |
| 31 | #endif |
| 32 | #if IS_ENABLED(CONFIG_CRYPTO_842) |
| 33 | "842", |
| 34 | #endif |
Sergey Senozhatsky | e46b8a0 | 2014-04-07 15:38:17 -0700 | [diff] [blame] | 35 | NULL |
| 36 | }; |
| 37 | |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 38 | static void zcomp_strm_free(struct zcomp_strm *zstrm) |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 39 | { |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 40 | if (!IS_ERR_OR_NULL(zstrm->tfm)) |
| 41 | crypto_free_comp(zstrm->tfm); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 42 | free_pages((unsigned long)zstrm->buffer, 1); |
| 43 | kfree(zstrm); |
| 44 | } |
| 45 | |
| 46 | /* |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 47 | * allocate new zcomp_strm structure with ->tfm initialized by |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 48 | * backend, return NULL on error |
| 49 | */ |
Sergey Senozhatsky | 16d3772 | 2016-07-26 15:22:59 -0700 | [diff] [blame] | 50 | static struct zcomp_strm *zcomp_strm_alloc(struct zcomp *comp) |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 51 | { |
Sergey Senozhatsky | 16d3772 | 2016-07-26 15:22:59 -0700 | [diff] [blame] | 52 | struct zcomp_strm *zstrm = kmalloc(sizeof(*zstrm), GFP_KERNEL); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 53 | if (!zstrm) |
| 54 | return NULL; |
| 55 | |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 56 | zstrm->tfm = crypto_alloc_comp(comp->name, 0, 0); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 57 | /* |
| 58 | * allocate 2 pages. 1 for compressed data, plus 1 extra for the |
| 59 | * case when compressed size is larger than the original one |
| 60 | */ |
Sergey Senozhatsky | 16d3772 | 2016-07-26 15:22:59 -0700 | [diff] [blame] | 61 | zstrm->buffer = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 62 | if (IS_ERR_OR_NULL(zstrm->tfm) || !zstrm->buffer) { |
| 63 | zcomp_strm_free(zstrm); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 64 | zstrm = NULL; |
| 65 | } |
| 66 | return zstrm; |
| 67 | } |
| 68 | |
Sergey Senozhatsky | 415403b | 2016-07-26 15:22:48 -0700 | [diff] [blame] | 69 | bool zcomp_available_algorithm(const char *comp) |
Sergey Senozhatsky | e46b8a0 | 2014-04-07 15:38:17 -0700 | [diff] [blame] | 70 | { |
Sergey Senozhatsky | e46b8a0 | 2014-04-07 15:38:17 -0700 | [diff] [blame] | 71 | int i = 0; |
| 72 | |
| 73 | while (backends[i]) { |
Sergey Senozhatsky | 415403b | 2016-07-26 15:22:48 -0700 | [diff] [blame] | 74 | if (sysfs_streq(comp, backends[i])) |
| 75 | return true; |
Sergey Senozhatsky | e46b8a0 | 2014-04-07 15:38:17 -0700 | [diff] [blame] | 76 | i++; |
| 77 | } |
Sergey Senozhatsky | 415403b | 2016-07-26 15:22:48 -0700 | [diff] [blame] | 78 | |
| 79 | /* |
| 80 | * Crypto does not ignore a trailing new line symbol, |
| 81 | * so make sure you don't supply a string containing |
| 82 | * one. |
| 83 | * This also means that we permit zcomp initialisation |
| 84 | * with any compressing algorithm known to crypto api. |
| 85 | */ |
| 86 | return crypto_has_comp(comp, 0, 0) == 1; |
Sergey Senozhatsky | e46b8a0 | 2014-04-07 15:38:17 -0700 | [diff] [blame] | 87 | } |
| 88 | |
Sergey Senozhatsky | 415403b | 2016-07-26 15:22:48 -0700 | [diff] [blame] | 89 | /* show available compressors */ |
| 90 | ssize_t zcomp_available_show(const char *comp, char *buf) |
Sergey Senozhatsky | d93435c | 2015-06-25 15:00:32 -0700 | [diff] [blame] | 91 | { |
Sergey Senozhatsky | 415403b | 2016-07-26 15:22:48 -0700 | [diff] [blame] | 92 | bool known_algorithm = false; |
| 93 | ssize_t sz = 0; |
| 94 | int i = 0; |
| 95 | |
| 96 | for (; backends[i]; i++) { |
| 97 | if (!strcmp(comp, backends[i])) { |
| 98 | known_algorithm = true; |
| 99 | sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, |
| 100 | "[%s] ", backends[i]); |
| 101 | } else { |
| 102 | sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, |
| 103 | "%s ", backends[i]); |
| 104 | } |
| 105 | } |
| 106 | |
| 107 | /* |
| 108 | * Out-of-tree module known to crypto api or a missing |
| 109 | * entry in `backends'. |
| 110 | */ |
| 111 | if (!known_algorithm && crypto_has_comp(comp, 0, 0) == 1) |
| 112 | sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, |
| 113 | "[%s] ", comp); |
| 114 | |
| 115 | sz += scnprintf(buf + sz, PAGE_SIZE - sz, "\n"); |
| 116 | return sz; |
Sergey Senozhatsky | d93435c | 2015-06-25 15:00:32 -0700 | [diff] [blame] | 117 | } |
| 118 | |
Sergey Senozhatsky | 2aea849 | 2016-07-26 15:22:42 -0700 | [diff] [blame] | 119 | struct zcomp_strm *zcomp_stream_get(struct zcomp *comp) |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 120 | { |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 121 | return *get_cpu_ptr(comp->stream); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 122 | } |
| 123 | |
Sergey Senozhatsky | 2aea849 | 2016-07-26 15:22:42 -0700 | [diff] [blame] | 124 | void zcomp_stream_put(struct zcomp *comp) |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 125 | { |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 126 | put_cpu_ptr(comp->stream); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 127 | } |
| 128 | |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 129 | int zcomp_compress(struct zcomp_strm *zstrm, |
| 130 | const void *src, unsigned int *dst_len) |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 131 | { |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 132 | /* |
| 133 | * Our dst memory (zstrm->buffer) is always `2 * PAGE_SIZE' sized |
| 134 | * because sometimes we can endup having a bigger compressed data |
| 135 | * due to various reasons: for example compression algorithms tend |
| 136 | * to add some padding to the compressed buffer. Speaking of padding, |
| 137 | * comp algorithm `842' pads the compressed length to multiple of 8 |
| 138 | * and returns -ENOSP when the dst memory is not big enough, which |
| 139 | * is not something that ZRAM wants to see. We can handle the |
| 140 | * `compressed_size > PAGE_SIZE' case easily in ZRAM, but when we |
| 141 | * receive -ERRNO from the compressing backend we can't help it |
| 142 | * anymore. To make `842' happy we need to tell the exact size of |
| 143 | * the dst buffer, zram_drv will take care of the fact that |
| 144 | * compressed buffer is too big. |
| 145 | */ |
| 146 | *dst_len = PAGE_SIZE * 2; |
| 147 | |
| 148 | return crypto_comp_compress(zstrm->tfm, |
| 149 | src, PAGE_SIZE, |
| 150 | zstrm->buffer, dst_len); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 151 | } |
| 152 | |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 153 | int zcomp_decompress(struct zcomp_strm *zstrm, |
| 154 | const void *src, unsigned int src_len, void *dst) |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 155 | { |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 156 | unsigned int dst_len = PAGE_SIZE; |
| 157 | |
| 158 | return crypto_comp_decompress(zstrm->tfm, |
| 159 | src, src_len, |
| 160 | dst, &dst_len); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 161 | } |
| 162 | |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 163 | static int __zcomp_cpu_notifier(struct zcomp *comp, |
| 164 | unsigned long action, unsigned long cpu) |
| 165 | { |
| 166 | struct zcomp_strm *zstrm; |
| 167 | |
| 168 | switch (action) { |
| 169 | case CPU_UP_PREPARE: |
| 170 | if (WARN_ON(*per_cpu_ptr(comp->stream, cpu))) |
| 171 | break; |
Sergey Senozhatsky | 16d3772 | 2016-07-26 15:22:59 -0700 | [diff] [blame] | 172 | zstrm = zcomp_strm_alloc(comp); |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 173 | if (IS_ERR_OR_NULL(zstrm)) { |
| 174 | pr_err("Can't allocate a compression stream\n"); |
| 175 | return NOTIFY_BAD; |
| 176 | } |
| 177 | *per_cpu_ptr(comp->stream, cpu) = zstrm; |
| 178 | break; |
| 179 | case CPU_DEAD: |
| 180 | case CPU_UP_CANCELED: |
| 181 | zstrm = *per_cpu_ptr(comp->stream, cpu); |
| 182 | if (!IS_ERR_OR_NULL(zstrm)) |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 183 | zcomp_strm_free(zstrm); |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 184 | *per_cpu_ptr(comp->stream, cpu) = NULL; |
| 185 | break; |
| 186 | default: |
| 187 | break; |
| 188 | } |
| 189 | return NOTIFY_OK; |
| 190 | } |
| 191 | |
| 192 | static int zcomp_cpu_notifier(struct notifier_block *nb, |
| 193 | unsigned long action, void *pcpu) |
| 194 | { |
| 195 | unsigned long cpu = (unsigned long)pcpu; |
| 196 | struct zcomp *comp = container_of(nb, typeof(*comp), notifier); |
| 197 | |
| 198 | return __zcomp_cpu_notifier(comp, action, cpu); |
| 199 | } |
| 200 | |
| 201 | static int zcomp_init(struct zcomp *comp) |
| 202 | { |
| 203 | unsigned long cpu; |
| 204 | int ret; |
| 205 | |
| 206 | comp->notifier.notifier_call = zcomp_cpu_notifier; |
| 207 | |
| 208 | comp->stream = alloc_percpu(struct zcomp_strm *); |
| 209 | if (!comp->stream) |
| 210 | return -ENOMEM; |
| 211 | |
| 212 | cpu_notifier_register_begin(); |
| 213 | for_each_online_cpu(cpu) { |
| 214 | ret = __zcomp_cpu_notifier(comp, CPU_UP_PREPARE, cpu); |
| 215 | if (ret == NOTIFY_BAD) |
| 216 | goto cleanup; |
| 217 | } |
| 218 | __register_cpu_notifier(&comp->notifier); |
| 219 | cpu_notifier_register_done(); |
| 220 | return 0; |
| 221 | |
| 222 | cleanup: |
| 223 | for_each_online_cpu(cpu) |
| 224 | __zcomp_cpu_notifier(comp, CPU_UP_CANCELED, cpu); |
| 225 | cpu_notifier_register_done(); |
| 226 | return -ENOMEM; |
| 227 | } |
| 228 | |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 229 | void zcomp_destroy(struct zcomp *comp) |
| 230 | { |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 231 | unsigned long cpu; |
| 232 | |
| 233 | cpu_notifier_register_begin(); |
| 234 | for_each_online_cpu(cpu) |
| 235 | __zcomp_cpu_notifier(comp, CPU_UP_CANCELED, cpu); |
| 236 | __unregister_cpu_notifier(&comp->notifier); |
| 237 | cpu_notifier_register_done(); |
| 238 | |
| 239 | free_percpu(comp->stream); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 240 | kfree(comp); |
| 241 | } |
| 242 | |
| 243 | /* |
| 244 | * search available compressors for requested algorithm. |
Sergey Senozhatsky | fcfa8d9 | 2014-04-07 15:38:20 -0700 | [diff] [blame] | 245 | * allocate new zcomp and initialize it. return compressing |
| 246 | * backend pointer or ERR_PTR if things went bad. ERR_PTR(-EINVAL) |
| 247 | * if requested algorithm is not supported, ERR_PTR(-ENOMEM) in |
Luis Henriques | 3aaf14d | 2015-09-17 16:01:40 -0700 | [diff] [blame] | 248 | * case of allocation error, or any other error potentially |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 249 | * returned by zcomp_init(). |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 250 | */ |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 251 | struct zcomp *zcomp_create(const char *compress) |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 252 | { |
| 253 | struct zcomp *comp; |
Luis Henriques | 3aaf14d | 2015-09-17 16:01:40 -0700 | [diff] [blame] | 254 | int error; |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 255 | |
Sergey Senozhatsky | 415403b | 2016-07-26 15:22:48 -0700 | [diff] [blame] | 256 | if (!zcomp_available_algorithm(compress)) |
Sergey Senozhatsky | fcfa8d9 | 2014-04-07 15:38:20 -0700 | [diff] [blame] | 257 | return ERR_PTR(-EINVAL); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 258 | |
| 259 | comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL); |
| 260 | if (!comp) |
Sergey Senozhatsky | fcfa8d9 | 2014-04-07 15:38:20 -0700 | [diff] [blame] | 261 | return ERR_PTR(-ENOMEM); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 262 | |
Sergey Senozhatsky | 415403b | 2016-07-26 15:22:48 -0700 | [diff] [blame] | 263 | comp->name = compress; |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 264 | error = zcomp_init(comp); |
Luis Henriques | 3aaf14d | 2015-09-17 16:01:40 -0700 | [diff] [blame] | 265 | if (error) { |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 266 | kfree(comp); |
Luis Henriques | 3aaf14d | 2015-09-17 16:01:40 -0700 | [diff] [blame] | 267 | return ERR_PTR(error); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 268 | } |
| 269 | return comp; |
| 270 | } |