Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2014 Sergey Senozhatsky. |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or |
| 5 | * modify it under the terms of the GNU General Public License |
| 6 | * as published by the Free Software Foundation; either version |
| 7 | * 2 of the License, or (at your option) any later version. |
| 8 | */ |
| 9 | |
| 10 | #include <linux/kernel.h> |
| 11 | #include <linux/string.h> |
Sergey Senozhatsky | fcfa8d9 | 2014-04-07 15:38:20 -0700 | [diff] [blame] | 12 | #include <linux/err.h> |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 13 | #include <linux/slab.h> |
| 14 | #include <linux/wait.h> |
| 15 | #include <linux/sched.h> |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 16 | #include <linux/cpu.h> |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 17 | #include <linux/crypto.h> |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 18 | |
| 19 | #include "zcomp.h" |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 20 | |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 21 | static const char * const backends[] = { |
| 22 | "lzo", |
Sergey Senozhatsky | ce1ed9f | 2016-07-26 15:22:54 -0700 | [diff] [blame] | 23 | #if IS_ENABLED(CONFIG_CRYPTO_LZ4) |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 24 | "lz4", |
Sergey Senozhatsky | 6e76668 | 2014-04-07 15:38:18 -0700 | [diff] [blame] | 25 | #endif |
Sergey Senozhatsky | eb9f56d | 2016-07-26 15:22:56 -0700 | [diff] [blame] | 26 | #if IS_ENABLED(CONFIG_CRYPTO_DEFLATE) |
| 27 | "deflate", |
| 28 | #endif |
| 29 | #if IS_ENABLED(CONFIG_CRYPTO_LZ4HC) |
| 30 | "lz4hc", |
| 31 | #endif |
| 32 | #if IS_ENABLED(CONFIG_CRYPTO_842) |
| 33 | "842", |
| 34 | #endif |
Sergey Senozhatsky | b3873ca | 2017-11-15 17:33:49 -0800 | [diff] [blame] | 35 | #if IS_ENABLED(CONFIG_CRYPTO_ZSTD) |
| 36 | "zstd", |
| 37 | #endif |
Sergey Senozhatsky | e46b8a0 | 2014-04-07 15:38:17 -0700 | [diff] [blame] | 38 | NULL |
| 39 | }; |
| 40 | |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 41 | static void zcomp_strm_free(struct zcomp_strm *zstrm) |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 42 | { |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 43 | if (!IS_ERR_OR_NULL(zstrm->tfm)) |
| 44 | crypto_free_comp(zstrm->tfm); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 45 | free_pages((unsigned long)zstrm->buffer, 1); |
| 46 | kfree(zstrm); |
| 47 | } |
| 48 | |
| 49 | /* |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 50 | * allocate new zcomp_strm structure with ->tfm initialized by |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 51 | * backend, return NULL on error |
| 52 | */ |
Sergey Senozhatsky | 16d3772 | 2016-07-26 15:22:59 -0700 | [diff] [blame] | 53 | static struct zcomp_strm *zcomp_strm_alloc(struct zcomp *comp) |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 54 | { |
Sergey Senozhatsky | 16d3772 | 2016-07-26 15:22:59 -0700 | [diff] [blame] | 55 | struct zcomp_strm *zstrm = kmalloc(sizeof(*zstrm), GFP_KERNEL); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 56 | if (!zstrm) |
| 57 | return NULL; |
| 58 | |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 59 | zstrm->tfm = crypto_alloc_comp(comp->name, 0, 0); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 60 | /* |
| 61 | * allocate 2 pages. 1 for compressed data, plus 1 extra for the |
| 62 | * case when compressed size is larger than the original one |
| 63 | */ |
Sergey Senozhatsky | 16d3772 | 2016-07-26 15:22:59 -0700 | [diff] [blame] | 64 | zstrm->buffer = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 65 | if (IS_ERR_OR_NULL(zstrm->tfm) || !zstrm->buffer) { |
| 66 | zcomp_strm_free(zstrm); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 67 | zstrm = NULL; |
| 68 | } |
| 69 | return zstrm; |
| 70 | } |
| 71 | |
Sergey Senozhatsky | 415403b | 2016-07-26 15:22:48 -0700 | [diff] [blame] | 72 | bool zcomp_available_algorithm(const char *comp) |
Sergey Senozhatsky | e46b8a0 | 2014-04-07 15:38:17 -0700 | [diff] [blame] | 73 | { |
Sergey Senozhatsky | e46b8a0 | 2014-04-07 15:38:17 -0700 | [diff] [blame] | 74 | int i = 0; |
| 75 | |
| 76 | while (backends[i]) { |
Sergey Senozhatsky | 415403b | 2016-07-26 15:22:48 -0700 | [diff] [blame] | 77 | if (sysfs_streq(comp, backends[i])) |
| 78 | return true; |
Sergey Senozhatsky | e46b8a0 | 2014-04-07 15:38:17 -0700 | [diff] [blame] | 79 | i++; |
| 80 | } |
Sergey Senozhatsky | 415403b | 2016-07-26 15:22:48 -0700 | [diff] [blame] | 81 | |
| 82 | /* |
| 83 | * Crypto does not ignore a trailing new line symbol, |
| 84 | * so make sure you don't supply a string containing |
| 85 | * one. |
| 86 | * This also means that we permit zcomp initialisation |
| 87 | * with any compressing algorithm known to crypto api. |
| 88 | */ |
| 89 | return crypto_has_comp(comp, 0, 0) == 1; |
Sergey Senozhatsky | e46b8a0 | 2014-04-07 15:38:17 -0700 | [diff] [blame] | 90 | } |
| 91 | |
Sergey Senozhatsky | 415403b | 2016-07-26 15:22:48 -0700 | [diff] [blame] | 92 | /* show available compressors */ |
| 93 | ssize_t zcomp_available_show(const char *comp, char *buf) |
Sergey Senozhatsky | d93435c | 2015-06-25 15:00:32 -0700 | [diff] [blame] | 94 | { |
Sergey Senozhatsky | 415403b | 2016-07-26 15:22:48 -0700 | [diff] [blame] | 95 | bool known_algorithm = false; |
| 96 | ssize_t sz = 0; |
| 97 | int i = 0; |
| 98 | |
| 99 | for (; backends[i]; i++) { |
| 100 | if (!strcmp(comp, backends[i])) { |
| 101 | known_algorithm = true; |
| 102 | sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, |
| 103 | "[%s] ", backends[i]); |
| 104 | } else { |
| 105 | sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, |
| 106 | "%s ", backends[i]); |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | /* |
| 111 | * Out-of-tree module known to crypto api or a missing |
| 112 | * entry in `backends'. |
| 113 | */ |
| 114 | if (!known_algorithm && crypto_has_comp(comp, 0, 0) == 1) |
| 115 | sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, |
| 116 | "[%s] ", comp); |
| 117 | |
| 118 | sz += scnprintf(buf + sz, PAGE_SIZE - sz, "\n"); |
| 119 | return sz; |
Sergey Senozhatsky | d93435c | 2015-06-25 15:00:32 -0700 | [diff] [blame] | 120 | } |
| 121 | |
Sergey Senozhatsky | 2aea849 | 2016-07-26 15:22:42 -0700 | [diff] [blame] | 122 | struct zcomp_strm *zcomp_stream_get(struct zcomp *comp) |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 123 | { |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 124 | return *get_cpu_ptr(comp->stream); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 125 | } |
| 126 | |
Sergey Senozhatsky | 2aea849 | 2016-07-26 15:22:42 -0700 | [diff] [blame] | 127 | void zcomp_stream_put(struct zcomp *comp) |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 128 | { |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 129 | put_cpu_ptr(comp->stream); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 130 | } |
| 131 | |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 132 | int zcomp_compress(struct zcomp_strm *zstrm, |
| 133 | const void *src, unsigned int *dst_len) |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 134 | { |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 135 | /* |
| 136 | * Our dst memory (zstrm->buffer) is always `2 * PAGE_SIZE' sized |
| 137 | * because sometimes we can endup having a bigger compressed data |
| 138 | * due to various reasons: for example compression algorithms tend |
| 139 | * to add some padding to the compressed buffer. Speaking of padding, |
| 140 | * comp algorithm `842' pads the compressed length to multiple of 8 |
| 141 | * and returns -ENOSP when the dst memory is not big enough, which |
| 142 | * is not something that ZRAM wants to see. We can handle the |
| 143 | * `compressed_size > PAGE_SIZE' case easily in ZRAM, but when we |
| 144 | * receive -ERRNO from the compressing backend we can't help it |
| 145 | * anymore. To make `842' happy we need to tell the exact size of |
| 146 | * the dst buffer, zram_drv will take care of the fact that |
| 147 | * compressed buffer is too big. |
| 148 | */ |
| 149 | *dst_len = PAGE_SIZE * 2; |
| 150 | |
| 151 | return crypto_comp_compress(zstrm->tfm, |
| 152 | src, PAGE_SIZE, |
| 153 | zstrm->buffer, dst_len); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 154 | } |
| 155 | |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 156 | int zcomp_decompress(struct zcomp_strm *zstrm, |
| 157 | const void *src, unsigned int src_len, void *dst) |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 158 | { |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 159 | unsigned int dst_len = PAGE_SIZE; |
| 160 | |
| 161 | return crypto_comp_decompress(zstrm->tfm, |
| 162 | src, src_len, |
| 163 | dst, &dst_len); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 164 | } |
| 165 | |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 166 | static int __zcomp_cpu_notifier(struct zcomp *comp, |
| 167 | unsigned long action, unsigned long cpu) |
| 168 | { |
| 169 | struct zcomp_strm *zstrm; |
| 170 | |
| 171 | switch (action) { |
| 172 | case CPU_UP_PREPARE: |
| 173 | if (WARN_ON(*per_cpu_ptr(comp->stream, cpu))) |
| 174 | break; |
Sergey Senozhatsky | 16d3772 | 2016-07-26 15:22:59 -0700 | [diff] [blame] | 175 | zstrm = zcomp_strm_alloc(comp); |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 176 | if (IS_ERR_OR_NULL(zstrm)) { |
| 177 | pr_err("Can't allocate a compression stream\n"); |
| 178 | return NOTIFY_BAD; |
| 179 | } |
| 180 | *per_cpu_ptr(comp->stream, cpu) = zstrm; |
| 181 | break; |
| 182 | case CPU_DEAD: |
| 183 | case CPU_UP_CANCELED: |
| 184 | zstrm = *per_cpu_ptr(comp->stream, cpu); |
| 185 | if (!IS_ERR_OR_NULL(zstrm)) |
Sergey Senozhatsky | ebaf9ab | 2016-07-26 15:22:45 -0700 | [diff] [blame] | 186 | zcomp_strm_free(zstrm); |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 187 | *per_cpu_ptr(comp->stream, cpu) = NULL; |
| 188 | break; |
| 189 | default: |
| 190 | break; |
| 191 | } |
| 192 | return NOTIFY_OK; |
| 193 | } |
| 194 | |
| 195 | static int zcomp_cpu_notifier(struct notifier_block *nb, |
| 196 | unsigned long action, void *pcpu) |
| 197 | { |
| 198 | unsigned long cpu = (unsigned long)pcpu; |
| 199 | struct zcomp *comp = container_of(nb, typeof(*comp), notifier); |
| 200 | |
| 201 | return __zcomp_cpu_notifier(comp, action, cpu); |
| 202 | } |
| 203 | |
| 204 | static int zcomp_init(struct zcomp *comp) |
| 205 | { |
| 206 | unsigned long cpu; |
| 207 | int ret; |
| 208 | |
| 209 | comp->notifier.notifier_call = zcomp_cpu_notifier; |
| 210 | |
| 211 | comp->stream = alloc_percpu(struct zcomp_strm *); |
| 212 | if (!comp->stream) |
| 213 | return -ENOMEM; |
| 214 | |
| 215 | cpu_notifier_register_begin(); |
| 216 | for_each_online_cpu(cpu) { |
| 217 | ret = __zcomp_cpu_notifier(comp, CPU_UP_PREPARE, cpu); |
| 218 | if (ret == NOTIFY_BAD) |
| 219 | goto cleanup; |
| 220 | } |
| 221 | __register_cpu_notifier(&comp->notifier); |
| 222 | cpu_notifier_register_done(); |
| 223 | return 0; |
| 224 | |
| 225 | cleanup: |
| 226 | for_each_online_cpu(cpu) |
| 227 | __zcomp_cpu_notifier(comp, CPU_UP_CANCELED, cpu); |
| 228 | cpu_notifier_register_done(); |
| 229 | return -ENOMEM; |
| 230 | } |
| 231 | |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 232 | void zcomp_destroy(struct zcomp *comp) |
| 233 | { |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 234 | unsigned long cpu; |
| 235 | |
| 236 | cpu_notifier_register_begin(); |
| 237 | for_each_online_cpu(cpu) |
| 238 | __zcomp_cpu_notifier(comp, CPU_UP_CANCELED, cpu); |
| 239 | __unregister_cpu_notifier(&comp->notifier); |
| 240 | cpu_notifier_register_done(); |
| 241 | |
| 242 | free_percpu(comp->stream); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 243 | kfree(comp); |
| 244 | } |
| 245 | |
| 246 | /* |
| 247 | * search available compressors for requested algorithm. |
Sergey Senozhatsky | fcfa8d9 | 2014-04-07 15:38:20 -0700 | [diff] [blame] | 248 | * allocate new zcomp and initialize it. return compressing |
| 249 | * backend pointer or ERR_PTR if things went bad. ERR_PTR(-EINVAL) |
| 250 | * if requested algorithm is not supported, ERR_PTR(-ENOMEM) in |
Luis Henriques | 3aaf14d | 2015-09-17 16:01:40 -0700 | [diff] [blame] | 251 | * case of allocation error, or any other error potentially |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 252 | * returned by zcomp_init(). |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 253 | */ |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 254 | struct zcomp *zcomp_create(const char *compress) |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 255 | { |
| 256 | struct zcomp *comp; |
Luis Henriques | 3aaf14d | 2015-09-17 16:01:40 -0700 | [diff] [blame] | 257 | int error; |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 258 | |
Sergey Senozhatsky | 415403b | 2016-07-26 15:22:48 -0700 | [diff] [blame] | 259 | if (!zcomp_available_algorithm(compress)) |
Sergey Senozhatsky | fcfa8d9 | 2014-04-07 15:38:20 -0700 | [diff] [blame] | 260 | return ERR_PTR(-EINVAL); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 261 | |
| 262 | comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL); |
| 263 | if (!comp) |
Sergey Senozhatsky | fcfa8d9 | 2014-04-07 15:38:20 -0700 | [diff] [blame] | 264 | return ERR_PTR(-ENOMEM); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 265 | |
Sergey Senozhatsky | 415403b | 2016-07-26 15:22:48 -0700 | [diff] [blame] | 266 | comp->name = compress; |
Sergey Senozhatsky | da9556a | 2016-05-20 16:59:51 -0700 | [diff] [blame] | 267 | error = zcomp_init(comp); |
Luis Henriques | 3aaf14d | 2015-09-17 16:01:40 -0700 | [diff] [blame] | 268 | if (error) { |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 269 | kfree(comp); |
Luis Henriques | 3aaf14d | 2015-09-17 16:01:40 -0700 | [diff] [blame] | 270 | return ERR_PTR(error); |
Sergey Senozhatsky | e7e1ef4 | 2014-04-07 15:38:11 -0700 | [diff] [blame] | 271 | } |
| 272 | return comp; |
| 273 | } |