Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 1 | /* |
| 2 | * Copyright(c) 2013-2015 Intel Corporation. All rights reserved. |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify |
| 5 | * it under the terms of version 2 of the GNU General Public License as |
| 6 | * published by the Free Software Foundation. |
| 7 | * |
| 8 | * This program is distributed in the hope that it will be useful, but |
| 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
| 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 11 | * General Public License for more details. |
| 12 | */ |
| 13 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
Dave Jiang | aa9ad44 | 2017-08-23 12:48:26 -0700 | [diff] [blame] | 14 | #include <linux/libnvdimm.h> |
Vishal Verma | 0930a75 | 2017-08-30 19:36:02 -0600 | [diff] [blame] | 15 | #include <linux/sched/mm.h> |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 16 | #include <linux/vmalloc.h> |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 17 | #include <linux/uaccess.h> |
Dan Williams | 3d88002 | 2015-05-31 15:02:11 -0400 | [diff] [blame] | 18 | #include <linux/module.h> |
Dan Williams | 8c2f7e8 | 2015-06-25 04:20:04 -0400 | [diff] [blame] | 19 | #include <linux/blkdev.h> |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 20 | #include <linux/fcntl.h> |
Dan Williams | e6dfb2d | 2015-04-25 03:56:17 -0400 | [diff] [blame] | 21 | #include <linux/async.h> |
Dan Williams | 8c2f7e8 | 2015-06-25 04:20:04 -0400 | [diff] [blame] | 22 | #include <linux/genhd.h> |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 23 | #include <linux/ndctl.h> |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 24 | #include <linux/sched.h> |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 25 | #include <linux/slab.h> |
| 26 | #include <linux/fs.h> |
| 27 | #include <linux/io.h> |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 28 | #include <linux/mm.h> |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 29 | #include <linux/nd.h> |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 30 | #include "nd-core.h" |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 31 | #include "nd.h" |
Dave Jiang | 006358b | 2017-04-07 15:33:31 -0700 | [diff] [blame] | 32 | #include "pfn.h" |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 33 | |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 34 | int nvdimm_major; |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 35 | static int nvdimm_bus_major; |
| 36 | static struct class *nd_class; |
Dan Williams | 1851594 | 2016-07-22 23:46:08 -0700 | [diff] [blame] | 37 | static DEFINE_IDA(nd_ida); |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 38 | |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 39 | static int to_nd_device_type(struct device *dev) |
| 40 | { |
| 41 | if (is_nvdimm(dev)) |
| 42 | return ND_DEVICE_DIMM; |
Dan Williams | c9e582a | 2017-05-29 23:12:19 -0700 | [diff] [blame] | 43 | else if (is_memory(dev)) |
Dan Williams | 3d88002 | 2015-05-31 15:02:11 -0400 | [diff] [blame] | 44 | return ND_DEVICE_REGION_PMEM; |
| 45 | else if (is_nd_blk(dev)) |
| 46 | return ND_DEVICE_REGION_BLK; |
Dan Williams | cd03412 | 2016-03-11 10:15:36 -0800 | [diff] [blame] | 47 | else if (is_nd_dax(dev)) |
| 48 | return ND_DEVICE_DAX_PMEM; |
Dan Williams | c9e582a | 2017-05-29 23:12:19 -0700 | [diff] [blame] | 49 | else if (is_nd_region(dev->parent)) |
Dan Williams | 3d88002 | 2015-05-31 15:02:11 -0400 | [diff] [blame] | 50 | return nd_region_to_nstype(to_nd_region(dev->parent)); |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 51 | |
| 52 | return 0; |
| 53 | } |
| 54 | |
| 55 | static int nvdimm_bus_uevent(struct device *dev, struct kobj_uevent_env *env) |
| 56 | { |
Toshi Kani | 41d7a6d | 2015-06-19 12:18:33 -0600 | [diff] [blame] | 57 | /* |
| 58 | * Ensure that region devices always have their numa node set as |
| 59 | * early as possible. |
| 60 | */ |
Dan Williams | c9e582a | 2017-05-29 23:12:19 -0700 | [diff] [blame] | 61 | if (is_nd_region(dev)) |
Toshi Kani | 41d7a6d | 2015-06-19 12:18:33 -0600 | [diff] [blame] | 62 | set_dev_node(dev, to_nd_region(dev)->numa_node); |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 63 | return add_uevent_var(env, "MODALIAS=" ND_DEVICE_MODALIAS_FMT, |
| 64 | to_nd_device_type(dev)); |
| 65 | } |
| 66 | |
Dan Williams | 3d88002 | 2015-05-31 15:02:11 -0400 | [diff] [blame] | 67 | static struct module *to_bus_provider(struct device *dev) |
| 68 | { |
| 69 | /* pin bus providers while regions are enabled */ |
Dan Williams | c9e582a | 2017-05-29 23:12:19 -0700 | [diff] [blame] | 70 | if (is_nd_region(dev)) { |
Dan Williams | 3d88002 | 2015-05-31 15:02:11 -0400 | [diff] [blame] | 71 | struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); |
| 72 | |
Dan Williams | bc9775d | 2016-07-21 20:03:19 -0700 | [diff] [blame] | 73 | return nvdimm_bus->nd_desc->module; |
Dan Williams | 3d88002 | 2015-05-31 15:02:11 -0400 | [diff] [blame] | 74 | } |
| 75 | return NULL; |
| 76 | } |
| 77 | |
Dan Williams | eaf9615 | 2015-05-01 13:11:27 -0400 | [diff] [blame] | 78 | static void nvdimm_bus_probe_start(struct nvdimm_bus *nvdimm_bus) |
| 79 | { |
| 80 | nvdimm_bus_lock(&nvdimm_bus->dev); |
| 81 | nvdimm_bus->probe_active++; |
| 82 | nvdimm_bus_unlock(&nvdimm_bus->dev); |
| 83 | } |
| 84 | |
| 85 | static void nvdimm_bus_probe_end(struct nvdimm_bus *nvdimm_bus) |
| 86 | { |
| 87 | nvdimm_bus_lock(&nvdimm_bus->dev); |
| 88 | if (--nvdimm_bus->probe_active == 0) |
Dan Williams | 1a547d2 | 2019-07-17 18:08:15 -0700 | [diff] [blame] | 89 | wake_up(&nvdimm_bus->wait); |
Dan Williams | eaf9615 | 2015-05-01 13:11:27 -0400 | [diff] [blame] | 90 | nvdimm_bus_unlock(&nvdimm_bus->dev); |
| 91 | } |
| 92 | |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 93 | static int nvdimm_bus_probe(struct device *dev) |
| 94 | { |
| 95 | struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver); |
Dan Williams | 3d88002 | 2015-05-31 15:02:11 -0400 | [diff] [blame] | 96 | struct module *provider = to_bus_provider(dev); |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 97 | struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); |
| 98 | int rc; |
| 99 | |
Dan Williams | 3d88002 | 2015-05-31 15:02:11 -0400 | [diff] [blame] | 100 | if (!try_module_get(provider)) |
| 101 | return -ENXIO; |
| 102 | |
Dan Williams | 3f46833 | 2018-06-01 14:10:58 -0700 | [diff] [blame] | 103 | dev_dbg(&nvdimm_bus->dev, "START: %s.probe(%s)\n", |
| 104 | dev->driver->name, dev_name(dev)); |
| 105 | |
Dan Williams | eaf9615 | 2015-05-01 13:11:27 -0400 | [diff] [blame] | 106 | nvdimm_bus_probe_start(nvdimm_bus); |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 107 | rc = nd_drv->probe(dev); |
Dan Williams | eaf9615 | 2015-05-01 13:11:27 -0400 | [diff] [blame] | 108 | if (rc == 0) |
| 109 | nd_region_probe_success(nvdimm_bus, dev); |
Dan Williams | bf9bccc | 2015-06-17 17:14:46 -0400 | [diff] [blame] | 110 | else |
| 111 | nd_region_disable(nvdimm_bus, dev); |
Dan Williams | eaf9615 | 2015-05-01 13:11:27 -0400 | [diff] [blame] | 112 | nvdimm_bus_probe_end(nvdimm_bus); |
| 113 | |
Dan Williams | 3f46833 | 2018-06-01 14:10:58 -0700 | [diff] [blame] | 114 | dev_dbg(&nvdimm_bus->dev, "END: %s.probe(%s) = %d\n", dev->driver->name, |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 115 | dev_name(dev), rc); |
Dan Williams | 8c2f7e8 | 2015-06-25 04:20:04 -0400 | [diff] [blame] | 116 | |
Dan Williams | 3d88002 | 2015-05-31 15:02:11 -0400 | [diff] [blame] | 117 | if (rc != 0) |
| 118 | module_put(provider); |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 119 | return rc; |
| 120 | } |
| 121 | |
| 122 | static int nvdimm_bus_remove(struct device *dev) |
| 123 | { |
| 124 | struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver); |
Dan Williams | 3d88002 | 2015-05-31 15:02:11 -0400 | [diff] [blame] | 125 | struct module *provider = to_bus_provider(dev); |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 126 | struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); |
Dan Williams | 6cf9c5b | 2016-05-18 09:13:13 -0700 | [diff] [blame] | 127 | int rc = 0; |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 128 | |
Dan Williams | 6cf9c5b | 2016-05-18 09:13:13 -0700 | [diff] [blame] | 129 | if (nd_drv->remove) |
| 130 | rc = nd_drv->remove(dev); |
Dan Williams | eaf9615 | 2015-05-01 13:11:27 -0400 | [diff] [blame] | 131 | nd_region_disable(nvdimm_bus, dev); |
| 132 | |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 133 | dev_dbg(&nvdimm_bus->dev, "%s.remove(%s) = %d\n", dev->driver->name, |
| 134 | dev_name(dev), rc); |
Dan Williams | 3d88002 | 2015-05-31 15:02:11 -0400 | [diff] [blame] | 135 | module_put(provider); |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 136 | return rc; |
| 137 | } |
| 138 | |
Dan Williams | 476f848 | 2016-07-09 00:12:52 -0700 | [diff] [blame] | 139 | static void nvdimm_bus_shutdown(struct device *dev) |
| 140 | { |
| 141 | struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); |
| 142 | struct nd_device_driver *nd_drv = NULL; |
| 143 | |
| 144 | if (dev->driver) |
| 145 | nd_drv = to_nd_device_driver(dev->driver); |
| 146 | |
| 147 | if (nd_drv && nd_drv->shutdown) { |
| 148 | nd_drv->shutdown(dev); |
| 149 | dev_dbg(&nvdimm_bus->dev, "%s.shutdown(%s)\n", |
| 150 | dev->driver->name, dev_name(dev)); |
| 151 | } |
| 152 | } |
| 153 | |
Dan Williams | 7199946 | 2016-02-18 10:29:49 -0800 | [diff] [blame] | 154 | void nd_device_notify(struct device *dev, enum nvdimm_event event) |
| 155 | { |
| 156 | device_lock(dev); |
| 157 | if (dev->driver) { |
| 158 | struct nd_device_driver *nd_drv; |
| 159 | |
| 160 | nd_drv = to_nd_device_driver(dev->driver); |
| 161 | if (nd_drv->notify) |
| 162 | nd_drv->notify(dev, event); |
| 163 | } |
| 164 | device_unlock(dev); |
| 165 | } |
| 166 | EXPORT_SYMBOL(nd_device_notify); |
| 167 | |
| 168 | void nvdimm_region_notify(struct nd_region *nd_region, enum nvdimm_event event) |
| 169 | { |
| 170 | struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev); |
| 171 | |
| 172 | if (!nvdimm_bus) |
| 173 | return; |
| 174 | |
| 175 | /* caller is responsible for holding a reference on the device */ |
| 176 | nd_device_notify(&nd_region->dev, event); |
| 177 | } |
| 178 | EXPORT_SYMBOL_GPL(nvdimm_region_notify); |
| 179 | |
Dan Williams | 23f4984 | 2017-04-29 15:24:03 -0700 | [diff] [blame] | 180 | struct clear_badblocks_context { |
| 181 | resource_size_t phys, cleared; |
| 182 | }; |
| 183 | |
| 184 | static int nvdimm_clear_badblocks_region(struct device *dev, void *data) |
| 185 | { |
| 186 | struct clear_badblocks_context *ctx = data; |
| 187 | struct nd_region *nd_region; |
| 188 | resource_size_t ndr_end; |
| 189 | sector_t sector; |
| 190 | |
| 191 | /* make sure device is a region */ |
Aneesh Kumar K.V | 2e93d24 | 2019-09-19 14:03:55 +0530 | [diff] [blame] | 192 | if (!is_memory(dev)) |
Dan Williams | 23f4984 | 2017-04-29 15:24:03 -0700 | [diff] [blame] | 193 | return 0; |
| 194 | |
| 195 | nd_region = to_nd_region(dev); |
| 196 | ndr_end = nd_region->ndr_start + nd_region->ndr_size - 1; |
| 197 | |
| 198 | /* make sure we are in the region */ |
| 199 | if (ctx->phys < nd_region->ndr_start |
| 200 | || (ctx->phys + ctx->cleared) > ndr_end) |
| 201 | return 0; |
| 202 | |
| 203 | sector = (ctx->phys - nd_region->ndr_start) / 512; |
| 204 | badblocks_clear(&nd_region->bb, sector, ctx->cleared / 512); |
| 205 | |
Toshi Kani | 975750a | 2017-06-12 16:25:11 -0600 | [diff] [blame] | 206 | if (nd_region->bb_state) |
| 207 | sysfs_notify_dirent(nd_region->bb_state); |
| 208 | |
Dan Williams | 23f4984 | 2017-04-29 15:24:03 -0700 | [diff] [blame] | 209 | return 0; |
| 210 | } |
| 211 | |
| 212 | static void nvdimm_clear_badblocks_regions(struct nvdimm_bus *nvdimm_bus, |
| 213 | phys_addr_t phys, u64 cleared) |
| 214 | { |
| 215 | struct clear_badblocks_context ctx = { |
| 216 | .phys = phys, |
| 217 | .cleared = cleared, |
| 218 | }; |
| 219 | |
| 220 | device_for_each_child(&nvdimm_bus->dev, &ctx, |
| 221 | nvdimm_clear_badblocks_region); |
| 222 | } |
| 223 | |
| 224 | static void nvdimm_account_cleared_poison(struct nvdimm_bus *nvdimm_bus, |
| 225 | phys_addr_t phys, u64 cleared) |
| 226 | { |
| 227 | if (cleared > 0) |
Dave Jiang | aa9ad44 | 2017-08-23 12:48:26 -0700 | [diff] [blame] | 228 | badrange_forget(&nvdimm_bus->badrange, phys, cleared); |
Dan Williams | 23f4984 | 2017-04-29 15:24:03 -0700 | [diff] [blame] | 229 | |
| 230 | if (cleared > 0 && cleared / 512) |
| 231 | nvdimm_clear_badblocks_regions(nvdimm_bus, phys, cleared); |
| 232 | } |
| 233 | |
Dan Williams | 59e6473 | 2016-03-08 07:16:07 -0800 | [diff] [blame] | 234 | long nvdimm_clear_poison(struct device *dev, phys_addr_t phys, |
| 235 | unsigned int len) |
| 236 | { |
| 237 | struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); |
| 238 | struct nvdimm_bus_descriptor *nd_desc; |
| 239 | struct nd_cmd_clear_error clear_err; |
| 240 | struct nd_cmd_ars_cap ars_cap; |
| 241 | u32 clear_err_unit, mask; |
Vishal Verma | 0930a75 | 2017-08-30 19:36:02 -0600 | [diff] [blame] | 242 | unsigned int noio_flag; |
Dan Williams | 59e6473 | 2016-03-08 07:16:07 -0800 | [diff] [blame] | 243 | int cmd_rc, rc; |
| 244 | |
| 245 | if (!nvdimm_bus) |
| 246 | return -ENXIO; |
| 247 | |
| 248 | nd_desc = nvdimm_bus->nd_desc; |
Dave Jiang | 1e8b8d9 | 2016-09-09 09:10:08 -0700 | [diff] [blame] | 249 | /* |
| 250 | * if ndctl does not exist, it's PMEM_LEGACY and |
| 251 | * we want to just pretend everything is handled. |
| 252 | */ |
Dan Williams | 59e6473 | 2016-03-08 07:16:07 -0800 | [diff] [blame] | 253 | if (!nd_desc->ndctl) |
Dave Jiang | 1e8b8d9 | 2016-09-09 09:10:08 -0700 | [diff] [blame] | 254 | return len; |
Dan Williams | 59e6473 | 2016-03-08 07:16:07 -0800 | [diff] [blame] | 255 | |
| 256 | memset(&ars_cap, 0, sizeof(ars_cap)); |
| 257 | ars_cap.address = phys; |
| 258 | ars_cap.length = len; |
Vishal Verma | 0930a75 | 2017-08-30 19:36:02 -0600 | [diff] [blame] | 259 | noio_flag = memalloc_noio_save(); |
Dan Williams | 59e6473 | 2016-03-08 07:16:07 -0800 | [diff] [blame] | 260 | rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, &ars_cap, |
| 261 | sizeof(ars_cap), &cmd_rc); |
Vishal Verma | 0930a75 | 2017-08-30 19:36:02 -0600 | [diff] [blame] | 262 | memalloc_noio_restore(noio_flag); |
Dan Williams | 59e6473 | 2016-03-08 07:16:07 -0800 | [diff] [blame] | 263 | if (rc < 0) |
| 264 | return rc; |
| 265 | if (cmd_rc < 0) |
| 266 | return cmd_rc; |
| 267 | clear_err_unit = ars_cap.clear_err_unit; |
| 268 | if (!clear_err_unit || !is_power_of_2(clear_err_unit)) |
| 269 | return -ENXIO; |
| 270 | |
| 271 | mask = clear_err_unit - 1; |
| 272 | if ((phys | len) & mask) |
| 273 | return -ENXIO; |
| 274 | memset(&clear_err, 0, sizeof(clear_err)); |
| 275 | clear_err.address = phys; |
| 276 | clear_err.length = len; |
Vishal Verma | 0930a75 | 2017-08-30 19:36:02 -0600 | [diff] [blame] | 277 | noio_flag = memalloc_noio_save(); |
Dan Williams | 59e6473 | 2016-03-08 07:16:07 -0800 | [diff] [blame] | 278 | rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_CLEAR_ERROR, &clear_err, |
| 279 | sizeof(clear_err), &cmd_rc); |
Vishal Verma | 0930a75 | 2017-08-30 19:36:02 -0600 | [diff] [blame] | 280 | memalloc_noio_restore(noio_flag); |
Dan Williams | 59e6473 | 2016-03-08 07:16:07 -0800 | [diff] [blame] | 281 | if (rc < 0) |
| 282 | return rc; |
| 283 | if (cmd_rc < 0) |
| 284 | return cmd_rc; |
Vishal Verma | e046114 | 2016-09-30 17:19:31 -0600 | [diff] [blame] | 285 | |
Dan Williams | 23f4984 | 2017-04-29 15:24:03 -0700 | [diff] [blame] | 286 | nvdimm_account_cleared_poison(nvdimm_bus, phys, clear_err.cleared); |
Toshi Kani | 8d13c02 | 2017-04-27 16:57:05 -0600 | [diff] [blame] | 287 | |
Dan Williams | 59e6473 | 2016-03-08 07:16:07 -0800 | [diff] [blame] | 288 | return clear_err.cleared; |
| 289 | } |
| 290 | EXPORT_SYMBOL_GPL(nvdimm_clear_poison); |
| 291 | |
Dan Williams | 1851594 | 2016-07-22 23:46:08 -0700 | [diff] [blame] | 292 | static int nvdimm_bus_match(struct device *dev, struct device_driver *drv); |
| 293 | |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 294 | static struct bus_type nvdimm_bus_type = { |
Dan Williams | e6dfb2d | 2015-04-25 03:56:17 -0400 | [diff] [blame] | 295 | .name = "nd", |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 296 | .uevent = nvdimm_bus_uevent, |
| 297 | .match = nvdimm_bus_match, |
| 298 | .probe = nvdimm_bus_probe, |
| 299 | .remove = nvdimm_bus_remove, |
Dan Williams | 476f848 | 2016-07-09 00:12:52 -0700 | [diff] [blame] | 300 | .shutdown = nvdimm_bus_shutdown, |
Dan Williams | e6dfb2d | 2015-04-25 03:56:17 -0400 | [diff] [blame] | 301 | }; |
| 302 | |
Dan Williams | 1851594 | 2016-07-22 23:46:08 -0700 | [diff] [blame] | 303 | static void nvdimm_bus_release(struct device *dev) |
| 304 | { |
| 305 | struct nvdimm_bus *nvdimm_bus; |
| 306 | |
| 307 | nvdimm_bus = container_of(dev, struct nvdimm_bus, dev); |
| 308 | ida_simple_remove(&nd_ida, nvdimm_bus->id); |
| 309 | kfree(nvdimm_bus); |
| 310 | } |
| 311 | |
| 312 | static bool is_nvdimm_bus(struct device *dev) |
| 313 | { |
| 314 | return dev->release == nvdimm_bus_release; |
| 315 | } |
| 316 | |
| 317 | struct nvdimm_bus *walk_to_nvdimm_bus(struct device *nd_dev) |
| 318 | { |
| 319 | struct device *dev; |
| 320 | |
| 321 | for (dev = nd_dev; dev; dev = dev->parent) |
| 322 | if (is_nvdimm_bus(dev)) |
| 323 | break; |
| 324 | dev_WARN_ONCE(nd_dev, !dev, "invalid dev, not on nd bus\n"); |
| 325 | if (dev) |
| 326 | return to_nvdimm_bus(dev); |
| 327 | return NULL; |
| 328 | } |
| 329 | |
| 330 | struct nvdimm_bus *to_nvdimm_bus(struct device *dev) |
| 331 | { |
| 332 | struct nvdimm_bus *nvdimm_bus; |
| 333 | |
| 334 | nvdimm_bus = container_of(dev, struct nvdimm_bus, dev); |
| 335 | WARN_ON(!is_nvdimm_bus(dev)); |
| 336 | return nvdimm_bus; |
| 337 | } |
| 338 | EXPORT_SYMBOL_GPL(to_nvdimm_bus); |
| 339 | |
| 340 | struct nvdimm_bus *nvdimm_bus_register(struct device *parent, |
| 341 | struct nvdimm_bus_descriptor *nd_desc) |
| 342 | { |
| 343 | struct nvdimm_bus *nvdimm_bus; |
| 344 | int rc; |
| 345 | |
| 346 | nvdimm_bus = kzalloc(sizeof(*nvdimm_bus), GFP_KERNEL); |
| 347 | if (!nvdimm_bus) |
| 348 | return NULL; |
| 349 | INIT_LIST_HEAD(&nvdimm_bus->list); |
| 350 | INIT_LIST_HEAD(&nvdimm_bus->mapping_list); |
Dan Williams | 1a547d2 | 2019-07-17 18:08:15 -0700 | [diff] [blame] | 351 | init_waitqueue_head(&nvdimm_bus->wait); |
Dan Williams | 1851594 | 2016-07-22 23:46:08 -0700 | [diff] [blame] | 352 | nvdimm_bus->id = ida_simple_get(&nd_ida, 0, 0, GFP_KERNEL); |
| 353 | mutex_init(&nvdimm_bus->reconfig_mutex); |
Dave Jiang | aa9ad44 | 2017-08-23 12:48:26 -0700 | [diff] [blame] | 354 | badrange_init(&nvdimm_bus->badrange); |
Dan Williams | 1851594 | 2016-07-22 23:46:08 -0700 | [diff] [blame] | 355 | if (nvdimm_bus->id < 0) { |
| 356 | kfree(nvdimm_bus); |
| 357 | return NULL; |
| 358 | } |
| 359 | nvdimm_bus->nd_desc = nd_desc; |
| 360 | nvdimm_bus->dev.parent = parent; |
| 361 | nvdimm_bus->dev.release = nvdimm_bus_release; |
| 362 | nvdimm_bus->dev.groups = nd_desc->attr_groups; |
| 363 | nvdimm_bus->dev.bus = &nvdimm_bus_type; |
Oliver O'Halloran | 1ff19f4 | 2018-04-06 15:21:13 +1000 | [diff] [blame] | 364 | nvdimm_bus->dev.of_node = nd_desc->of_node; |
Dan Williams | 1851594 | 2016-07-22 23:46:08 -0700 | [diff] [blame] | 365 | dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id); |
| 366 | rc = device_register(&nvdimm_bus->dev); |
| 367 | if (rc) { |
| 368 | dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc); |
| 369 | goto err; |
| 370 | } |
| 371 | |
| 372 | return nvdimm_bus; |
| 373 | err: |
| 374 | put_device(&nvdimm_bus->dev); |
| 375 | return NULL; |
| 376 | } |
| 377 | EXPORT_SYMBOL_GPL(nvdimm_bus_register); |
| 378 | |
| 379 | void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus) |
| 380 | { |
| 381 | if (!nvdimm_bus) |
| 382 | return; |
| 383 | device_unregister(&nvdimm_bus->dev); |
| 384 | } |
| 385 | EXPORT_SYMBOL_GPL(nvdimm_bus_unregister); |
| 386 | |
| 387 | static int child_unregister(struct device *dev, void *data) |
| 388 | { |
| 389 | /* |
| 390 | * the singular ndctl class device per bus needs to be |
| 391 | * "device_destroy"ed, so skip it here |
| 392 | * |
| 393 | * i.e. remove classless children |
| 394 | */ |
| 395 | if (dev->class) |
| 396 | /* pass */; |
| 397 | else |
| 398 | nd_device_unregister(dev, ND_SYNC); |
| 399 | return 0; |
| 400 | } |
| 401 | |
Dave Jiang | aa9ad44 | 2017-08-23 12:48:26 -0700 | [diff] [blame] | 402 | static void free_badrange_list(struct list_head *badrange_list) |
Dan Williams | 1851594 | 2016-07-22 23:46:08 -0700 | [diff] [blame] | 403 | { |
Dave Jiang | aa9ad44 | 2017-08-23 12:48:26 -0700 | [diff] [blame] | 404 | struct badrange_entry *bre, *next; |
Dan Williams | 1851594 | 2016-07-22 23:46:08 -0700 | [diff] [blame] | 405 | |
Dave Jiang | aa9ad44 | 2017-08-23 12:48:26 -0700 | [diff] [blame] | 406 | list_for_each_entry_safe(bre, next, badrange_list, list) { |
| 407 | list_del(&bre->list); |
| 408 | kfree(bre); |
Dan Williams | 1851594 | 2016-07-22 23:46:08 -0700 | [diff] [blame] | 409 | } |
Dave Jiang | aa9ad44 | 2017-08-23 12:48:26 -0700 | [diff] [blame] | 410 | list_del_init(badrange_list); |
Dan Williams | 1851594 | 2016-07-22 23:46:08 -0700 | [diff] [blame] | 411 | } |
| 412 | |
| 413 | static int nd_bus_remove(struct device *dev) |
| 414 | { |
| 415 | struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); |
| 416 | |
| 417 | mutex_lock(&nvdimm_bus_list_mutex); |
| 418 | list_del_init(&nvdimm_bus->list); |
| 419 | mutex_unlock(&nvdimm_bus_list_mutex); |
| 420 | |
Dan Williams | 1a547d2 | 2019-07-17 18:08:15 -0700 | [diff] [blame] | 421 | wait_event(nvdimm_bus->wait, |
| 422 | atomic_read(&nvdimm_bus->ioctl_active) == 0); |
| 423 | |
Dan Williams | 1851594 | 2016-07-22 23:46:08 -0700 | [diff] [blame] | 424 | nd_synchronize(); |
| 425 | device_for_each_child(&nvdimm_bus->dev, NULL, child_unregister); |
| 426 | |
Dave Jiang | aa9ad44 | 2017-08-23 12:48:26 -0700 | [diff] [blame] | 427 | spin_lock(&nvdimm_bus->badrange.lock); |
| 428 | free_badrange_list(&nvdimm_bus->badrange.list); |
| 429 | spin_unlock(&nvdimm_bus->badrange.lock); |
Dan Williams | 1851594 | 2016-07-22 23:46:08 -0700 | [diff] [blame] | 430 | |
| 431 | nvdimm_bus_destroy_ndctl(nvdimm_bus); |
| 432 | |
| 433 | return 0; |
| 434 | } |
| 435 | |
| 436 | static int nd_bus_probe(struct device *dev) |
| 437 | { |
| 438 | struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); |
| 439 | int rc; |
| 440 | |
| 441 | rc = nvdimm_bus_create_ndctl(nvdimm_bus); |
| 442 | if (rc) |
| 443 | return rc; |
| 444 | |
| 445 | mutex_lock(&nvdimm_bus_list_mutex); |
| 446 | list_add_tail(&nvdimm_bus->list, &nvdimm_bus_list); |
| 447 | mutex_unlock(&nvdimm_bus_list_mutex); |
| 448 | |
| 449 | /* enable bus provider attributes to look up their local context */ |
| 450 | dev_set_drvdata(dev, nvdimm_bus->nd_desc); |
| 451 | |
| 452 | return 0; |
| 453 | } |
| 454 | |
| 455 | static struct nd_device_driver nd_bus_driver = { |
| 456 | .probe = nd_bus_probe, |
| 457 | .remove = nd_bus_remove, |
| 458 | .drv = { |
| 459 | .name = "nd_bus", |
| 460 | .suppress_bind_attrs = true, |
| 461 | .bus = &nvdimm_bus_type, |
| 462 | .owner = THIS_MODULE, |
| 463 | .mod_name = KBUILD_MODNAME, |
| 464 | }, |
| 465 | }; |
| 466 | |
| 467 | static int nvdimm_bus_match(struct device *dev, struct device_driver *drv) |
| 468 | { |
| 469 | struct nd_device_driver *nd_drv = to_nd_device_driver(drv); |
| 470 | |
| 471 | if (is_nvdimm_bus(dev) && nd_drv == &nd_bus_driver) |
| 472 | return true; |
| 473 | |
| 474 | return !!test_bit(to_nd_device_type(dev), &nd_drv->type); |
| 475 | } |
| 476 | |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 477 | static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain); |
| 478 | |
| 479 | void nd_synchronize(void) |
| 480 | { |
| 481 | async_synchronize_full_domain(&nd_async_domain); |
| 482 | } |
| 483 | EXPORT_SYMBOL_GPL(nd_synchronize); |
| 484 | |
| 485 | static void nd_async_device_register(void *d, async_cookie_t cookie) |
| 486 | { |
| 487 | struct device *dev = d; |
| 488 | |
| 489 | if (device_add(dev) != 0) { |
| 490 | dev_err(dev, "%s: failed\n", __func__); |
| 491 | put_device(dev); |
| 492 | } |
| 493 | put_device(dev); |
Alexander Duyck | 4f1a55a | 2018-09-25 13:53:02 -0700 | [diff] [blame] | 494 | if (dev->parent) |
| 495 | put_device(dev->parent); |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 496 | } |
| 497 | |
| 498 | static void nd_async_device_unregister(void *d, async_cookie_t cookie) |
| 499 | { |
| 500 | struct device *dev = d; |
| 501 | |
Dan Williams | 0ba1c63 | 2015-05-30 12:35:36 -0400 | [diff] [blame] | 502 | /* flush bus operations before delete */ |
| 503 | nvdimm_bus_lock(dev); |
| 504 | nvdimm_bus_unlock(dev); |
| 505 | |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 506 | device_unregister(dev); |
| 507 | put_device(dev); |
| 508 | } |
| 509 | |
Dan Williams | 8c2f7e8 | 2015-06-25 04:20:04 -0400 | [diff] [blame] | 510 | void __nd_device_register(struct device *dev) |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 511 | { |
Dan Williams | cd03412 | 2016-03-11 10:15:36 -0800 | [diff] [blame] | 512 | if (!dev) |
| 513 | return; |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 514 | dev->bus = &nvdimm_bus_type; |
Alexander Duyck | 4f1a55a | 2018-09-25 13:53:02 -0700 | [diff] [blame] | 515 | if (dev->parent) |
| 516 | get_device(dev->parent); |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 517 | get_device(dev); |
| 518 | async_schedule_domain(nd_async_device_register, dev, |
| 519 | &nd_async_domain); |
| 520 | } |
Dan Williams | 8c2f7e8 | 2015-06-25 04:20:04 -0400 | [diff] [blame] | 521 | |
| 522 | void nd_device_register(struct device *dev) |
| 523 | { |
| 524 | device_initialize(dev); |
| 525 | __nd_device_register(dev); |
| 526 | } |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 527 | EXPORT_SYMBOL(nd_device_register); |
| 528 | |
| 529 | void nd_device_unregister(struct device *dev, enum nd_async_mode mode) |
| 530 | { |
Dan Williams | d16bbdb | 2019-08-05 18:31:56 -0700 | [diff] [blame] | 531 | bool killed; |
| 532 | |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 533 | switch (mode) { |
| 534 | case ND_ASYNC: |
Dan Williams | d16bbdb | 2019-08-05 18:31:56 -0700 | [diff] [blame] | 535 | /* |
| 536 | * In the async case this is being triggered with the |
| 537 | * device lock held and the unregistration work needs to |
| 538 | * be moved out of line iff this is thread has won the |
| 539 | * race to schedule the deletion. |
| 540 | */ |
| 541 | if (!kill_device(dev)) |
| 542 | return; |
| 543 | |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 544 | get_device(dev); |
| 545 | async_schedule_domain(nd_async_device_unregister, dev, |
| 546 | &nd_async_domain); |
| 547 | break; |
| 548 | case ND_SYNC: |
Dan Williams | d16bbdb | 2019-08-05 18:31:56 -0700 | [diff] [blame] | 549 | /* |
| 550 | * In the sync case the device is being unregistered due |
| 551 | * to a state change of the parent. Claim the kill state |
| 552 | * to synchronize against other unregistration requests, |
| 553 | * or otherwise let the async path handle it if the |
| 554 | * unregistration was already queued. |
| 555 | */ |
| 556 | device_lock(dev); |
| 557 | killed = kill_device(dev); |
| 558 | device_unlock(dev); |
| 559 | |
| 560 | if (!killed) |
| 561 | return; |
| 562 | |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 563 | nd_synchronize(); |
| 564 | device_unregister(dev); |
| 565 | break; |
| 566 | } |
| 567 | } |
| 568 | EXPORT_SYMBOL(nd_device_unregister); |
| 569 | |
| 570 | /** |
| 571 | * __nd_driver_register() - register a region or a namespace driver |
| 572 | * @nd_drv: driver to register |
| 573 | * @owner: automatically set by nd_driver_register() macro |
| 574 | * @mod_name: automatically set by nd_driver_register() macro |
| 575 | */ |
| 576 | int __nd_driver_register(struct nd_device_driver *nd_drv, struct module *owner, |
| 577 | const char *mod_name) |
| 578 | { |
| 579 | struct device_driver *drv = &nd_drv->drv; |
| 580 | |
| 581 | if (!nd_drv->type) { |
| 582 | pr_debug("driver type bitmask not set (%pf)\n", |
| 583 | __builtin_return_address(0)); |
| 584 | return -EINVAL; |
| 585 | } |
| 586 | |
Dan Williams | 6cf9c5b | 2016-05-18 09:13:13 -0700 | [diff] [blame] | 587 | if (!nd_drv->probe) { |
| 588 | pr_debug("%s ->probe() must be specified\n", mod_name); |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 589 | return -EINVAL; |
| 590 | } |
| 591 | |
| 592 | drv->bus = &nvdimm_bus_type; |
| 593 | drv->owner = owner; |
| 594 | drv->mod_name = mod_name; |
| 595 | |
| 596 | return driver_register(drv); |
| 597 | } |
| 598 | EXPORT_SYMBOL(__nd_driver_register); |
| 599 | |
Dan Williams | 5813882 | 2015-06-23 20:08:34 -0400 | [diff] [blame] | 600 | int nvdimm_revalidate_disk(struct gendisk *disk) |
| 601 | { |
Dan Williams | 52c44d9 | 2016-06-15 19:43:07 -0700 | [diff] [blame] | 602 | struct device *dev = disk_to_dev(disk)->parent; |
Dan Williams | 5813882 | 2015-06-23 20:08:34 -0400 | [diff] [blame] | 603 | struct nd_region *nd_region = to_nd_region(dev->parent); |
Robert Elliott | 254a4cd | 2018-05-31 18:36:36 -0500 | [diff] [blame] | 604 | int disk_ro = get_disk_ro(disk); |
Dan Williams | 5813882 | 2015-06-23 20:08:34 -0400 | [diff] [blame] | 605 | |
Robert Elliott | 254a4cd | 2018-05-31 18:36:36 -0500 | [diff] [blame] | 606 | /* |
| 607 | * Upgrade to read-only if the region is read-only preserve as |
| 608 | * read-only if the disk is already read-only. |
| 609 | */ |
| 610 | if (disk_ro || nd_region->ro == disk_ro) |
Dan Williams | 5813882 | 2015-06-23 20:08:34 -0400 | [diff] [blame] | 611 | return 0; |
| 612 | |
Robert Elliott | 254a4cd | 2018-05-31 18:36:36 -0500 | [diff] [blame] | 613 | dev_info(dev, "%s read-only, marking %s read-only\n", |
| 614 | dev_name(&nd_region->dev), disk->disk_name); |
| 615 | set_disk_ro(disk, 1); |
Dan Williams | 5813882 | 2015-06-23 20:08:34 -0400 | [diff] [blame] | 616 | |
| 617 | return 0; |
| 618 | |
| 619 | } |
| 620 | EXPORT_SYMBOL(nvdimm_revalidate_disk); |
| 621 | |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 622 | static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, |
| 623 | char *buf) |
| 624 | { |
| 625 | return sprintf(buf, ND_DEVICE_MODALIAS_FMT "\n", |
| 626 | to_nd_device_type(dev)); |
| 627 | } |
| 628 | static DEVICE_ATTR_RO(modalias); |
| 629 | |
| 630 | static ssize_t devtype_show(struct device *dev, struct device_attribute *attr, |
| 631 | char *buf) |
| 632 | { |
| 633 | return sprintf(buf, "%s\n", dev->type->name); |
| 634 | } |
| 635 | static DEVICE_ATTR_RO(devtype); |
| 636 | |
| 637 | static struct attribute *nd_device_attributes[] = { |
| 638 | &dev_attr_modalias.attr, |
| 639 | &dev_attr_devtype.attr, |
| 640 | NULL, |
| 641 | }; |
| 642 | |
Qian Cai | 90a5645 | 2019-05-16 12:04:53 -0400 | [diff] [blame] | 643 | /* |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 644 | * nd_device_attribute_group - generic attributes for all devices on an nd bus |
| 645 | */ |
| 646 | struct attribute_group nd_device_attribute_group = { |
| 647 | .attrs = nd_device_attributes, |
| 648 | }; |
| 649 | EXPORT_SYMBOL_GPL(nd_device_attribute_group); |
| 650 | |
Toshi Kani | 74ae66c | 2015-06-19 12:18:34 -0600 | [diff] [blame] | 651 | static ssize_t numa_node_show(struct device *dev, |
| 652 | struct device_attribute *attr, char *buf) |
| 653 | { |
| 654 | return sprintf(buf, "%d\n", dev_to_node(dev)); |
| 655 | } |
| 656 | static DEVICE_ATTR_RO(numa_node); |
| 657 | |
| 658 | static struct attribute *nd_numa_attributes[] = { |
| 659 | &dev_attr_numa_node.attr, |
| 660 | NULL, |
| 661 | }; |
| 662 | |
| 663 | static umode_t nd_numa_attr_visible(struct kobject *kobj, struct attribute *a, |
| 664 | int n) |
| 665 | { |
| 666 | if (!IS_ENABLED(CONFIG_NUMA)) |
| 667 | return 0; |
| 668 | |
| 669 | return a->mode; |
| 670 | } |
| 671 | |
Qian Cai | 90a5645 | 2019-05-16 12:04:53 -0400 | [diff] [blame] | 672 | /* |
Toshi Kani | 74ae66c | 2015-06-19 12:18:34 -0600 | [diff] [blame] | 673 | * nd_numa_attribute_group - NUMA attributes for all devices on an nd bus |
| 674 | */ |
| 675 | struct attribute_group nd_numa_attribute_group = { |
| 676 | .attrs = nd_numa_attributes, |
| 677 | .is_visible = nd_numa_attr_visible, |
| 678 | }; |
| 679 | EXPORT_SYMBOL_GPL(nd_numa_attribute_group); |
| 680 | |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 681 | int nvdimm_bus_create_ndctl(struct nvdimm_bus *nvdimm_bus) |
| 682 | { |
| 683 | dev_t devt = MKDEV(nvdimm_bus_major, nvdimm_bus->id); |
| 684 | struct device *dev; |
| 685 | |
| 686 | dev = device_create(nd_class, &nvdimm_bus->dev, devt, nvdimm_bus, |
| 687 | "ndctl%d", nvdimm_bus->id); |
| 688 | |
Dan Williams | 4258895 | 2016-05-27 13:28:31 -0700 | [diff] [blame] | 689 | if (IS_ERR(dev)) |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 690 | dev_dbg(&nvdimm_bus->dev, "failed to register ndctl%d: %ld\n", |
| 691 | nvdimm_bus->id, PTR_ERR(dev)); |
Dan Williams | 4258895 | 2016-05-27 13:28:31 -0700 | [diff] [blame] | 692 | return PTR_ERR_OR_ZERO(dev); |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 693 | } |
| 694 | |
| 695 | void nvdimm_bus_destroy_ndctl(struct nvdimm_bus *nvdimm_bus) |
| 696 | { |
| 697 | device_destroy(nd_class, MKDEV(nvdimm_bus_major, nvdimm_bus->id)); |
| 698 | } |
| 699 | |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 700 | static const struct nd_cmd_desc __nd_cmd_dimm_descs[] = { |
| 701 | [ND_CMD_IMPLEMENTED] = { }, |
| 702 | [ND_CMD_SMART] = { |
| 703 | .out_num = 2, |
Dan Williams | 2112911 | 2016-04-07 19:58:44 -0700 | [diff] [blame] | 704 | .out_sizes = { 4, 128, }, |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 705 | }, |
| 706 | [ND_CMD_SMART_THRESHOLD] = { |
| 707 | .out_num = 2, |
| 708 | .out_sizes = { 4, 8, }, |
| 709 | }, |
| 710 | [ND_CMD_DIMM_FLAGS] = { |
| 711 | .out_num = 2, |
| 712 | .out_sizes = { 4, 4 }, |
| 713 | }, |
| 714 | [ND_CMD_GET_CONFIG_SIZE] = { |
| 715 | .out_num = 3, |
| 716 | .out_sizes = { 4, 4, 4, }, |
| 717 | }, |
| 718 | [ND_CMD_GET_CONFIG_DATA] = { |
| 719 | .in_num = 2, |
| 720 | .in_sizes = { 4, 4, }, |
| 721 | .out_num = 2, |
| 722 | .out_sizes = { 4, UINT_MAX, }, |
| 723 | }, |
| 724 | [ND_CMD_SET_CONFIG_DATA] = { |
| 725 | .in_num = 3, |
| 726 | .in_sizes = { 4, 4, UINT_MAX, }, |
| 727 | .out_num = 1, |
| 728 | .out_sizes = { 4, }, |
| 729 | }, |
| 730 | [ND_CMD_VENDOR] = { |
| 731 | .in_num = 3, |
| 732 | .in_sizes = { 4, 4, UINT_MAX, }, |
| 733 | .out_num = 3, |
| 734 | .out_sizes = { 4, 4, UINT_MAX, }, |
| 735 | }, |
Dan Williams | 31eca76 | 2016-04-28 16:23:43 -0700 | [diff] [blame] | 736 | [ND_CMD_CALL] = { |
| 737 | .in_num = 2, |
| 738 | .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, }, |
| 739 | .out_num = 1, |
| 740 | .out_sizes = { UINT_MAX, }, |
| 741 | }, |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 742 | }; |
| 743 | |
| 744 | const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd) |
| 745 | { |
| 746 | if (cmd < ARRAY_SIZE(__nd_cmd_dimm_descs)) |
| 747 | return &__nd_cmd_dimm_descs[cmd]; |
| 748 | return NULL; |
| 749 | } |
| 750 | EXPORT_SYMBOL_GPL(nd_cmd_dimm_desc); |
| 751 | |
| 752 | static const struct nd_cmd_desc __nd_cmd_bus_descs[] = { |
| 753 | [ND_CMD_IMPLEMENTED] = { }, |
| 754 | [ND_CMD_ARS_CAP] = { |
| 755 | .in_num = 2, |
| 756 | .in_sizes = { 8, 8, }, |
Dan Williams | 4577b06 | 2016-02-17 13:08:58 -0800 | [diff] [blame] | 757 | .out_num = 4, |
| 758 | .out_sizes = { 4, 4, 4, 4, }, |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 759 | }, |
| 760 | [ND_CMD_ARS_START] = { |
Dan Williams | 4577b06 | 2016-02-17 13:08:58 -0800 | [diff] [blame] | 761 | .in_num = 5, |
| 762 | .in_sizes = { 8, 8, 2, 1, 5, }, |
| 763 | .out_num = 2, |
| 764 | .out_sizes = { 4, 4, }, |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 765 | }, |
| 766 | [ND_CMD_ARS_STATUS] = { |
Dan Williams | 747ffe1 | 2016-02-19 15:21:14 -0800 | [diff] [blame] | 767 | .out_num = 3, |
| 768 | .out_sizes = { 4, 4, UINT_MAX, }, |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 769 | }, |
Dan Williams | d4f3236 | 2016-03-03 16:08:54 -0800 | [diff] [blame] | 770 | [ND_CMD_CLEAR_ERROR] = { |
| 771 | .in_num = 2, |
| 772 | .in_sizes = { 8, 8, }, |
| 773 | .out_num = 3, |
| 774 | .out_sizes = { 4, 4, 8, }, |
| 775 | }, |
Dan Williams | 31eca76 | 2016-04-28 16:23:43 -0700 | [diff] [blame] | 776 | [ND_CMD_CALL] = { |
| 777 | .in_num = 2, |
| 778 | .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, }, |
| 779 | .out_num = 1, |
| 780 | .out_sizes = { UINT_MAX, }, |
| 781 | }, |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 782 | }; |
| 783 | |
| 784 | const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd) |
| 785 | { |
| 786 | if (cmd < ARRAY_SIZE(__nd_cmd_bus_descs)) |
| 787 | return &__nd_cmd_bus_descs[cmd]; |
| 788 | return NULL; |
| 789 | } |
| 790 | EXPORT_SYMBOL_GPL(nd_cmd_bus_desc); |
| 791 | |
| 792 | u32 nd_cmd_in_size(struct nvdimm *nvdimm, int cmd, |
| 793 | const struct nd_cmd_desc *desc, int idx, void *buf) |
| 794 | { |
| 795 | if (idx >= desc->in_num) |
| 796 | return UINT_MAX; |
| 797 | |
| 798 | if (desc->in_sizes[idx] < UINT_MAX) |
| 799 | return desc->in_sizes[idx]; |
| 800 | |
| 801 | if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA && idx == 2) { |
| 802 | struct nd_cmd_set_config_hdr *hdr = buf; |
| 803 | |
| 804 | return hdr->in_length; |
| 805 | } else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2) { |
| 806 | struct nd_cmd_vendor_hdr *hdr = buf; |
| 807 | |
| 808 | return hdr->in_length; |
Dan Williams | 31eca76 | 2016-04-28 16:23:43 -0700 | [diff] [blame] | 809 | } else if (cmd == ND_CMD_CALL) { |
| 810 | struct nd_cmd_pkg *pkg = buf; |
| 811 | |
| 812 | return pkg->nd_size_in; |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 813 | } |
| 814 | |
| 815 | return UINT_MAX; |
| 816 | } |
| 817 | EXPORT_SYMBOL_GPL(nd_cmd_in_size); |
| 818 | |
| 819 | u32 nd_cmd_out_size(struct nvdimm *nvdimm, int cmd, |
| 820 | const struct nd_cmd_desc *desc, int idx, const u32 *in_field, |
Dan Williams | efda1b5d | 2016-12-06 09:10:12 -0800 | [diff] [blame] | 821 | const u32 *out_field, unsigned long remainder) |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 822 | { |
| 823 | if (idx >= desc->out_num) |
| 824 | return UINT_MAX; |
| 825 | |
| 826 | if (desc->out_sizes[idx] < UINT_MAX) |
| 827 | return desc->out_sizes[idx]; |
| 828 | |
| 829 | if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && idx == 1) |
| 830 | return in_field[1]; |
| 831 | else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2) |
| 832 | return out_field[1]; |
Dan Williams | efda1b5d | 2016-12-06 09:10:12 -0800 | [diff] [blame] | 833 | else if (!nvdimm && cmd == ND_CMD_ARS_STATUS && idx == 2) { |
| 834 | /* |
| 835 | * Per table 9-276 ARS Data in ACPI 6.1, out_field[1] is |
| 836 | * "Size of Output Buffer in bytes, including this |
| 837 | * field." |
| 838 | */ |
| 839 | if (out_field[1] < 4) |
| 840 | return 0; |
| 841 | /* |
| 842 | * ACPI 6.1 is ambiguous if 'status' is included in the |
| 843 | * output size. If we encounter an output size that |
| 844 | * overshoots the remainder by 4 bytes, assume it was |
| 845 | * including 'status'. |
| 846 | */ |
Vishal Verma | 286e877 | 2018-08-10 13:23:15 -0600 | [diff] [blame] | 847 | if (out_field[1] - 4 == remainder) |
Dan Williams | efda1b5d | 2016-12-06 09:10:12 -0800 | [diff] [blame] | 848 | return remainder; |
Vishal Verma | 286e877 | 2018-08-10 13:23:15 -0600 | [diff] [blame] | 849 | return out_field[1] - 8; |
Dan Williams | efda1b5d | 2016-12-06 09:10:12 -0800 | [diff] [blame] | 850 | } else if (cmd == ND_CMD_CALL) { |
Dan Williams | 31eca76 | 2016-04-28 16:23:43 -0700 | [diff] [blame] | 851 | struct nd_cmd_pkg *pkg = (struct nd_cmd_pkg *) in_field; |
| 852 | |
| 853 | return pkg->nd_size_out; |
| 854 | } |
| 855 | |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 856 | |
| 857 | return UINT_MAX; |
| 858 | } |
| 859 | EXPORT_SYMBOL_GPL(nd_cmd_out_size); |
| 860 | |
Dan Williams | bf9bccc | 2015-06-17 17:14:46 -0400 | [diff] [blame] | 861 | void wait_nvdimm_bus_probe_idle(struct device *dev) |
Dan Williams | eaf9615 | 2015-05-01 13:11:27 -0400 | [diff] [blame] | 862 | { |
Dan Williams | bf9bccc | 2015-06-17 17:14:46 -0400 | [diff] [blame] | 863 | struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); |
| 864 | |
Dan Williams | eaf9615 | 2015-05-01 13:11:27 -0400 | [diff] [blame] | 865 | do { |
| 866 | if (nvdimm_bus->probe_active == 0) |
| 867 | break; |
Dan Williams | 2364ed0 | 2019-08-05 18:32:13 -0700 | [diff] [blame] | 868 | nvdimm_bus_unlock(dev); |
| 869 | device_unlock(dev); |
Dan Williams | 1a547d2 | 2019-07-17 18:08:15 -0700 | [diff] [blame] | 870 | wait_event(nvdimm_bus->wait, |
Dan Williams | eaf9615 | 2015-05-01 13:11:27 -0400 | [diff] [blame] | 871 | nvdimm_bus->probe_active == 0); |
Dan Williams | 2364ed0 | 2019-08-05 18:32:13 -0700 | [diff] [blame] | 872 | device_lock(dev); |
| 873 | nvdimm_bus_lock(dev); |
Dan Williams | eaf9615 | 2015-05-01 13:11:27 -0400 | [diff] [blame] | 874 | } while (true); |
| 875 | } |
| 876 | |
Dave Jiang | 006358b | 2017-04-07 15:33:31 -0700 | [diff] [blame] | 877 | static int nd_pmem_forget_poison_check(struct device *dev, void *data) |
Dan Williams | d4f3236 | 2016-03-03 16:08:54 -0800 | [diff] [blame] | 878 | { |
Dave Jiang | 006358b | 2017-04-07 15:33:31 -0700 | [diff] [blame] | 879 | struct nd_cmd_clear_error *clear_err = |
| 880 | (struct nd_cmd_clear_error *)data; |
| 881 | struct nd_btt *nd_btt = is_nd_btt(dev) ? to_nd_btt(dev) : NULL; |
| 882 | struct nd_pfn *nd_pfn = is_nd_pfn(dev) ? to_nd_pfn(dev) : NULL; |
| 883 | struct nd_dax *nd_dax = is_nd_dax(dev) ? to_nd_dax(dev) : NULL; |
| 884 | struct nd_namespace_common *ndns = NULL; |
| 885 | struct nd_namespace_io *nsio; |
| 886 | resource_size_t offset = 0, end_trunc = 0, start, end, pstart, pend; |
| 887 | |
| 888 | if (nd_dax || !dev->driver) |
| 889 | return 0; |
| 890 | |
| 891 | start = clear_err->address; |
| 892 | end = clear_err->address + clear_err->cleared - 1; |
| 893 | |
| 894 | if (nd_btt || nd_pfn || nd_dax) { |
| 895 | if (nd_btt) |
| 896 | ndns = nd_btt->ndns; |
| 897 | else if (nd_pfn) |
| 898 | ndns = nd_pfn->ndns; |
| 899 | else if (nd_dax) |
| 900 | ndns = nd_dax->nd_pfn.ndns; |
| 901 | |
| 902 | if (!ndns) |
| 903 | return 0; |
| 904 | } else |
| 905 | ndns = to_ndns(dev); |
| 906 | |
| 907 | nsio = to_nd_namespace_io(&ndns->dev); |
| 908 | pstart = nsio->res.start + offset; |
| 909 | pend = nsio->res.end - end_trunc; |
| 910 | |
| 911 | if ((pstart >= start) && (pend <= end)) |
Dan Williams | d4f3236 | 2016-03-03 16:08:54 -0800 | [diff] [blame] | 912 | return -EBUSY; |
Dave Jiang | 006358b | 2017-04-07 15:33:31 -0700 | [diff] [blame] | 913 | |
Dan Williams | d4f3236 | 2016-03-03 16:08:54 -0800 | [diff] [blame] | 914 | return 0; |
Dave Jiang | 006358b | 2017-04-07 15:33:31 -0700 | [diff] [blame] | 915 | |
| 916 | } |
| 917 | |
| 918 | static int nd_ns_forget_poison_check(struct device *dev, void *data) |
| 919 | { |
| 920 | return device_for_each_child(dev, data, nd_pmem_forget_poison_check); |
Dan Williams | d4f3236 | 2016-03-03 16:08:54 -0800 | [diff] [blame] | 921 | } |
| 922 | |
Dan Williams | eaf9615 | 2015-05-01 13:11:27 -0400 | [diff] [blame] | 923 | /* set_config requires an idle interleave set */ |
Dan Williams | 87bf572 | 2016-02-22 21:50:31 -0800 | [diff] [blame] | 924 | static int nd_cmd_clear_to_send(struct nvdimm_bus *nvdimm_bus, |
Dave Jiang | 006358b | 2017-04-07 15:33:31 -0700 | [diff] [blame] | 925 | struct nvdimm *nvdimm, unsigned int cmd, void *data) |
Dan Williams | eaf9615 | 2015-05-01 13:11:27 -0400 | [diff] [blame] | 926 | { |
Dan Williams | 87bf572 | 2016-02-22 21:50:31 -0800 | [diff] [blame] | 927 | struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc; |
| 928 | |
| 929 | /* ask the bus provider if it would like to block this request */ |
| 930 | if (nd_desc->clear_to_send) { |
| 931 | int rc = nd_desc->clear_to_send(nd_desc, nvdimm, cmd); |
| 932 | |
| 933 | if (rc) |
| 934 | return rc; |
| 935 | } |
Dan Williams | eaf9615 | 2015-05-01 13:11:27 -0400 | [diff] [blame] | 936 | |
Dan Williams | d4f3236 | 2016-03-03 16:08:54 -0800 | [diff] [blame] | 937 | /* require clear error to go through the pmem driver */ |
| 938 | if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR) |
Dave Jiang | 006358b | 2017-04-07 15:33:31 -0700 | [diff] [blame] | 939 | return device_for_each_child(&nvdimm_bus->dev, data, |
| 940 | nd_ns_forget_poison_check); |
Dan Williams | d4f3236 | 2016-03-03 16:08:54 -0800 | [diff] [blame] | 941 | |
Dan Williams | eaf9615 | 2015-05-01 13:11:27 -0400 | [diff] [blame] | 942 | if (!nvdimm || cmd != ND_CMD_SET_CONFIG_DATA) |
| 943 | return 0; |
| 944 | |
Dan Williams | 87bf572 | 2016-02-22 21:50:31 -0800 | [diff] [blame] | 945 | /* prevent label manipulation while the kernel owns label updates */ |
Dan Williams | bf9bccc | 2015-06-17 17:14:46 -0400 | [diff] [blame] | 946 | wait_nvdimm_bus_probe_idle(&nvdimm_bus->dev); |
Dan Williams | eaf9615 | 2015-05-01 13:11:27 -0400 | [diff] [blame] | 947 | if (atomic_read(&nvdimm->busy)) |
| 948 | return -EBUSY; |
| 949 | return 0; |
| 950 | } |
| 951 | |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 952 | static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm, |
| 953 | int read_only, unsigned int ioctl_cmd, unsigned long arg) |
| 954 | { |
| 955 | struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc; |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 956 | const struct nd_cmd_desc *desc = NULL; |
| 957 | unsigned int cmd = _IOC_NR(ioctl_cmd); |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 958 | struct device *dev = &nvdimm_bus->dev; |
Dan Williams | 58738c4 | 2017-08-31 15:41:55 -0700 | [diff] [blame] | 959 | void __user *p = (void __user *) arg; |
Dan Williams | 7f000e7 | 2019-08-05 18:32:07 -0700 | [diff] [blame] | 960 | char *out_env = NULL, *in_env = NULL; |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 961 | const char *cmd_name, *dimm_name; |
Dan Williams | 58738c4 | 2017-08-31 15:41:55 -0700 | [diff] [blame] | 962 | u32 in_len = 0, out_len = 0; |
| 963 | unsigned int func = cmd; |
Dan Williams | e3654ec | 2016-04-28 16:17:07 -0700 | [diff] [blame] | 964 | unsigned long cmd_mask; |
Dan Williams | 58738c4 | 2017-08-31 15:41:55 -0700 | [diff] [blame] | 965 | struct nd_cmd_pkg pkg; |
Dave Jiang | 006358b | 2017-04-07 15:33:31 -0700 | [diff] [blame] | 966 | int rc, i, cmd_rc; |
Dan Williams | 7f000e7 | 2019-08-05 18:32:07 -0700 | [diff] [blame] | 967 | void *buf = NULL; |
Dan Williams | 58738c4 | 2017-08-31 15:41:55 -0700 | [diff] [blame] | 968 | u64 buf_len = 0; |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 969 | |
| 970 | if (nvdimm) { |
| 971 | desc = nd_cmd_dimm_desc(cmd); |
| 972 | cmd_name = nvdimm_cmd_name(cmd); |
Dan Williams | e3654ec | 2016-04-28 16:17:07 -0700 | [diff] [blame] | 973 | cmd_mask = nvdimm->cmd_mask; |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 974 | dimm_name = dev_name(&nvdimm->dev); |
| 975 | } else { |
| 976 | desc = nd_cmd_bus_desc(cmd); |
| 977 | cmd_name = nvdimm_bus_cmd_name(cmd); |
Dan Williams | e3654ec | 2016-04-28 16:17:07 -0700 | [diff] [blame] | 978 | cmd_mask = nd_desc->cmd_mask; |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 979 | dimm_name = "bus"; |
| 980 | } |
| 981 | |
Dan Williams | 31eca76 | 2016-04-28 16:23:43 -0700 | [diff] [blame] | 982 | if (cmd == ND_CMD_CALL) { |
| 983 | if (copy_from_user(&pkg, p, sizeof(pkg))) |
| 984 | return -EFAULT; |
| 985 | } |
| 986 | |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 987 | if (!desc || (desc->out_num + desc->in_num == 0) || |
Dan Williams | e3654ec | 2016-04-28 16:17:07 -0700 | [diff] [blame] | 988 | !test_bit(cmd, &cmd_mask)) |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 989 | return -ENOTTY; |
| 990 | |
| 991 | /* fail write commands (when read-only) */ |
| 992 | if (read_only) |
Jerry Hoemann | 07accfa | 2016-01-06 16:03:41 -0700 | [diff] [blame] | 993 | switch (cmd) { |
| 994 | case ND_CMD_VENDOR: |
| 995 | case ND_CMD_SET_CONFIG_DATA: |
| 996 | case ND_CMD_ARS_START: |
Dan Williams | d4f3236 | 2016-03-03 16:08:54 -0800 | [diff] [blame] | 997 | case ND_CMD_CLEAR_ERROR: |
Dan Williams | 31eca76 | 2016-04-28 16:23:43 -0700 | [diff] [blame] | 998 | case ND_CMD_CALL: |
Dan Williams | 2364ed0 | 2019-08-05 18:32:13 -0700 | [diff] [blame] | 999 | dev_dbg(dev, "'%s' command while read-only.\n", |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1000 | nvdimm ? nvdimm_cmd_name(cmd) |
| 1001 | : nvdimm_bus_cmd_name(cmd)); |
| 1002 | return -EPERM; |
| 1003 | default: |
| 1004 | break; |
| 1005 | } |
| 1006 | |
| 1007 | /* process an input envelope */ |
Dan Williams | 7f000e7 | 2019-08-05 18:32:07 -0700 | [diff] [blame] | 1008 | in_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL); |
| 1009 | if (!in_env) |
| 1010 | return -ENOMEM; |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1011 | for (i = 0; i < desc->in_num; i++) { |
| 1012 | u32 in_size, copy; |
| 1013 | |
| 1014 | in_size = nd_cmd_in_size(nvdimm, cmd, desc, i, in_env); |
| 1015 | if (in_size == UINT_MAX) { |
| 1016 | dev_err(dev, "%s:%s unknown input size cmd: %s field: %d\n", |
| 1017 | __func__, dimm_name, cmd_name, i); |
Dan Williams | 7f000e7 | 2019-08-05 18:32:07 -0700 | [diff] [blame] | 1018 | rc = -ENXIO; |
| 1019 | goto out; |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1020 | } |
Dan Williams | 7f000e7 | 2019-08-05 18:32:07 -0700 | [diff] [blame] | 1021 | if (in_len < ND_CMD_MAX_ENVELOPE) |
| 1022 | copy = min_t(u32, ND_CMD_MAX_ENVELOPE - in_len, in_size); |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1023 | else |
| 1024 | copy = 0; |
Dan Williams | 7f000e7 | 2019-08-05 18:32:07 -0700 | [diff] [blame] | 1025 | if (copy && copy_from_user(&in_env[in_len], p + in_len, copy)) { |
| 1026 | rc = -EFAULT; |
| 1027 | goto out; |
| 1028 | } |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1029 | in_len += in_size; |
| 1030 | } |
| 1031 | |
Dan Williams | 31eca76 | 2016-04-28 16:23:43 -0700 | [diff] [blame] | 1032 | if (cmd == ND_CMD_CALL) { |
Jerry Hoemann | 53b85a4 | 2017-06-30 20:41:22 -0700 | [diff] [blame] | 1033 | func = pkg.nd_command; |
Dan Williams | 426824d | 2018-03-05 16:39:31 -0800 | [diff] [blame] | 1034 | dev_dbg(dev, "%s, idx: %llu, in: %u, out: %u, len %llu\n", |
| 1035 | dimm_name, pkg.nd_command, |
Dan Williams | 31eca76 | 2016-04-28 16:23:43 -0700 | [diff] [blame] | 1036 | in_len, out_len, buf_len); |
Dan Williams | 31eca76 | 2016-04-28 16:23:43 -0700 | [diff] [blame] | 1037 | } |
| 1038 | |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1039 | /* process an output envelope */ |
Dan Williams | 7f000e7 | 2019-08-05 18:32:07 -0700 | [diff] [blame] | 1040 | out_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL); |
| 1041 | if (!out_env) { |
| 1042 | rc = -ENOMEM; |
| 1043 | goto out; |
| 1044 | } |
| 1045 | |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1046 | for (i = 0; i < desc->out_num; i++) { |
| 1047 | u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, |
Dan Williams | efda1b5d | 2016-12-06 09:10:12 -0800 | [diff] [blame] | 1048 | (u32 *) in_env, (u32 *) out_env, 0); |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1049 | u32 copy; |
| 1050 | |
| 1051 | if (out_size == UINT_MAX) { |
Dan Williams | 426824d | 2018-03-05 16:39:31 -0800 | [diff] [blame] | 1052 | dev_dbg(dev, "%s unknown output size cmd: %s field: %d\n", |
| 1053 | dimm_name, cmd_name, i); |
Dan Williams | 7f000e7 | 2019-08-05 18:32:07 -0700 | [diff] [blame] | 1054 | rc = -EFAULT; |
| 1055 | goto out; |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1056 | } |
Dan Williams | 7f000e7 | 2019-08-05 18:32:07 -0700 | [diff] [blame] | 1057 | if (out_len < ND_CMD_MAX_ENVELOPE) |
| 1058 | copy = min_t(u32, ND_CMD_MAX_ENVELOPE - out_len, out_size); |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1059 | else |
| 1060 | copy = 0; |
| 1061 | if (copy && copy_from_user(&out_env[out_len], |
Dan Williams | 7f000e7 | 2019-08-05 18:32:07 -0700 | [diff] [blame] | 1062 | p + in_len + out_len, copy)) { |
| 1063 | rc = -EFAULT; |
| 1064 | goto out; |
| 1065 | } |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1066 | out_len += out_size; |
| 1067 | } |
| 1068 | |
Dan Williams | 58738c4 | 2017-08-31 15:41:55 -0700 | [diff] [blame] | 1069 | buf_len = (u64) out_len + (u64) in_len; |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1070 | if (buf_len > ND_IOCTL_MAX_BUFLEN) { |
Dan Williams | 426824d | 2018-03-05 16:39:31 -0800 | [diff] [blame] | 1071 | dev_dbg(dev, "%s cmd: %s buf_len: %llu > %d\n", dimm_name, |
| 1072 | cmd_name, buf_len, ND_IOCTL_MAX_BUFLEN); |
Dan Williams | 7f000e7 | 2019-08-05 18:32:07 -0700 | [diff] [blame] | 1073 | rc = -EINVAL; |
| 1074 | goto out; |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1075 | } |
| 1076 | |
| 1077 | buf = vmalloc(buf_len); |
Dan Williams | 7f000e7 | 2019-08-05 18:32:07 -0700 | [diff] [blame] | 1078 | if (!buf) { |
| 1079 | rc = -ENOMEM; |
| 1080 | goto out; |
| 1081 | } |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1082 | |
| 1083 | if (copy_from_user(buf, p, buf_len)) { |
| 1084 | rc = -EFAULT; |
| 1085 | goto out; |
| 1086 | } |
| 1087 | |
Dan Williams | 2364ed0 | 2019-08-05 18:32:13 -0700 | [diff] [blame] | 1088 | device_lock(dev); |
| 1089 | nvdimm_bus_lock(dev); |
Jerry Hoemann | 53b85a4 | 2017-06-30 20:41:22 -0700 | [diff] [blame] | 1090 | rc = nd_cmd_clear_to_send(nvdimm_bus, nvdimm, func, buf); |
Dan Williams | eaf9615 | 2015-05-01 13:11:27 -0400 | [diff] [blame] | 1091 | if (rc) |
| 1092 | goto out_unlock; |
| 1093 | |
Dave Jiang | 006358b | 2017-04-07 15:33:31 -0700 | [diff] [blame] | 1094 | rc = nd_desc->ndctl(nd_desc, nvdimm, cmd, buf, buf_len, &cmd_rc); |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1095 | if (rc < 0) |
Dan Williams | eaf9615 | 2015-05-01 13:11:27 -0400 | [diff] [blame] | 1096 | goto out_unlock; |
Dave Jiang | 006358b | 2017-04-07 15:33:31 -0700 | [diff] [blame] | 1097 | |
| 1098 | if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR && cmd_rc >= 0) { |
| 1099 | struct nd_cmd_clear_error *clear_err = buf; |
Dave Jiang | 006358b | 2017-04-07 15:33:31 -0700 | [diff] [blame] | 1100 | |
Dan Williams | 23f4984 | 2017-04-29 15:24:03 -0700 | [diff] [blame] | 1101 | nvdimm_account_cleared_poison(nvdimm_bus, clear_err->address, |
| 1102 | clear_err->cleared); |
Dave Jiang | 006358b | 2017-04-07 15:33:31 -0700 | [diff] [blame] | 1103 | } |
Dan Williams | 0beb201 | 2017-04-07 09:47:24 -0700 | [diff] [blame] | 1104 | |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1105 | if (copy_to_user(p, buf, buf_len)) |
| 1106 | rc = -EFAULT; |
Dan Williams | 0beb201 | 2017-04-07 09:47:24 -0700 | [diff] [blame] | 1107 | |
Dan Williams | 7f000e7 | 2019-08-05 18:32:07 -0700 | [diff] [blame] | 1108 | out_unlock: |
Dan Williams | 2364ed0 | 2019-08-05 18:32:13 -0700 | [diff] [blame] | 1109 | nvdimm_bus_unlock(dev); |
| 1110 | device_unlock(dev); |
Dan Williams | 7f000e7 | 2019-08-05 18:32:07 -0700 | [diff] [blame] | 1111 | out: |
| 1112 | kfree(in_env); |
| 1113 | kfree(out_env); |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1114 | vfree(buf); |
| 1115 | return rc; |
| 1116 | } |
| 1117 | |
Dan Williams | 1a547d2 | 2019-07-17 18:08:15 -0700 | [diff] [blame] | 1118 | enum nd_ioctl_mode { |
| 1119 | BUS_IOCTL, |
| 1120 | DIMM_IOCTL, |
| 1121 | }; |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1122 | |
| 1123 | static int match_dimm(struct device *dev, void *data) |
| 1124 | { |
| 1125 | long id = (long) data; |
| 1126 | |
| 1127 | if (is_nvdimm(dev)) { |
| 1128 | struct nvdimm *nvdimm = to_nvdimm(dev); |
| 1129 | |
| 1130 | return nvdimm->id == id; |
| 1131 | } |
| 1132 | |
| 1133 | return 0; |
| 1134 | } |
| 1135 | |
Dan Williams | 1a547d2 | 2019-07-17 18:08:15 -0700 | [diff] [blame] | 1136 | static long nd_ioctl(struct file *file, unsigned int cmd, unsigned long arg, |
| 1137 | enum nd_ioctl_mode mode) |
| 1138 | |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1139 | { |
Dan Williams | 1a547d2 | 2019-07-17 18:08:15 -0700 | [diff] [blame] | 1140 | struct nvdimm_bus *nvdimm_bus, *found = NULL; |
| 1141 | long id = (long) file->private_data; |
| 1142 | struct nvdimm *nvdimm = NULL; |
| 1143 | int rc, ro; |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1144 | |
Jerry Hoemann | 4dc0e7be | 2016-01-06 16:03:39 -0700 | [diff] [blame] | 1145 | ro = ((file->f_flags & O_ACCMODE) == O_RDONLY); |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1146 | mutex_lock(&nvdimm_bus_list_mutex); |
| 1147 | list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) { |
Dan Williams | 1a547d2 | 2019-07-17 18:08:15 -0700 | [diff] [blame] | 1148 | if (mode == DIMM_IOCTL) { |
| 1149 | struct device *dev; |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1150 | |
Dan Williams | 1a547d2 | 2019-07-17 18:08:15 -0700 | [diff] [blame] | 1151 | dev = device_find_child(&nvdimm_bus->dev, |
| 1152 | file->private_data, match_dimm); |
| 1153 | if (!dev) |
| 1154 | continue; |
| 1155 | nvdimm = to_nvdimm(dev); |
| 1156 | found = nvdimm_bus; |
| 1157 | } else if (nvdimm_bus->id == id) { |
| 1158 | found = nvdimm_bus; |
| 1159 | } |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1160 | |
Dan Williams | 1a547d2 | 2019-07-17 18:08:15 -0700 | [diff] [blame] | 1161 | if (found) { |
| 1162 | atomic_inc(&nvdimm_bus->ioctl_active); |
| 1163 | break; |
| 1164 | } |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1165 | } |
| 1166 | mutex_unlock(&nvdimm_bus_list_mutex); |
| 1167 | |
Dan Williams | 1a547d2 | 2019-07-17 18:08:15 -0700 | [diff] [blame] | 1168 | if (!found) |
| 1169 | return -ENXIO; |
| 1170 | |
| 1171 | nvdimm_bus = found; |
| 1172 | rc = __nd_ioctl(nvdimm_bus, nvdimm, ro, cmd, arg); |
| 1173 | |
| 1174 | if (nvdimm) |
| 1175 | put_device(&nvdimm->dev); |
| 1176 | if (atomic_dec_and_test(&nvdimm_bus->ioctl_active)) |
| 1177 | wake_up(&nvdimm_bus->wait); |
| 1178 | |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1179 | return rc; |
| 1180 | } |
| 1181 | |
Dan Williams | 1a547d2 | 2019-07-17 18:08:15 -0700 | [diff] [blame] | 1182 | static long bus_ioctl(struct file *file, unsigned int cmd, unsigned long arg) |
| 1183 | { |
| 1184 | return nd_ioctl(file, cmd, arg, BUS_IOCTL); |
| 1185 | } |
| 1186 | |
| 1187 | static long dimm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) |
| 1188 | { |
| 1189 | return nd_ioctl(file, cmd, arg, DIMM_IOCTL); |
| 1190 | } |
| 1191 | |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1192 | static int nd_open(struct inode *inode, struct file *file) |
| 1193 | { |
| 1194 | long minor = iminor(inode); |
| 1195 | |
| 1196 | file->private_data = (void *) minor; |
| 1197 | return 0; |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 1198 | } |
| 1199 | |
| 1200 | static const struct file_operations nvdimm_bus_fops = { |
| 1201 | .owner = THIS_MODULE, |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1202 | .open = nd_open, |
Dan Williams | 1a547d2 | 2019-07-17 18:08:15 -0700 | [diff] [blame] | 1203 | .unlocked_ioctl = bus_ioctl, |
| 1204 | .compat_ioctl = bus_ioctl, |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 1205 | .llseek = noop_llseek, |
| 1206 | }; |
| 1207 | |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1208 | static const struct file_operations nvdimm_fops = { |
| 1209 | .owner = THIS_MODULE, |
| 1210 | .open = nd_open, |
Dan Williams | 1a547d2 | 2019-07-17 18:08:15 -0700 | [diff] [blame] | 1211 | .unlocked_ioctl = dimm_ioctl, |
| 1212 | .compat_ioctl = dimm_ioctl, |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1213 | .llseek = noop_llseek, |
| 1214 | }; |
| 1215 | |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 1216 | int __init nvdimm_bus_init(void) |
| 1217 | { |
| 1218 | int rc; |
| 1219 | |
Dan Williams | e6dfb2d | 2015-04-25 03:56:17 -0400 | [diff] [blame] | 1220 | rc = bus_register(&nvdimm_bus_type); |
| 1221 | if (rc) |
| 1222 | return rc; |
| 1223 | |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 1224 | rc = register_chrdev(0, "ndctl", &nvdimm_bus_fops); |
| 1225 | if (rc < 0) |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1226 | goto err_bus_chrdev; |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 1227 | nvdimm_bus_major = rc; |
| 1228 | |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1229 | rc = register_chrdev(0, "dimmctl", &nvdimm_fops); |
| 1230 | if (rc < 0) |
| 1231 | goto err_dimm_chrdev; |
| 1232 | nvdimm_major = rc; |
| 1233 | |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 1234 | nd_class = class_create(THIS_MODULE, "nd"); |
Axel Lin | daa1dee | 2015-06-28 17:00:57 +0800 | [diff] [blame] | 1235 | if (IS_ERR(nd_class)) { |
| 1236 | rc = PTR_ERR(nd_class); |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 1237 | goto err_class; |
Axel Lin | daa1dee | 2015-06-28 17:00:57 +0800 | [diff] [blame] | 1238 | } |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 1239 | |
Dan Williams | 1851594 | 2016-07-22 23:46:08 -0700 | [diff] [blame] | 1240 | rc = driver_register(&nd_bus_driver.drv); |
| 1241 | if (rc) |
| 1242 | goto err_nd_bus; |
| 1243 | |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 1244 | return 0; |
| 1245 | |
Dan Williams | 1851594 | 2016-07-22 23:46:08 -0700 | [diff] [blame] | 1246 | err_nd_bus: |
| 1247 | class_destroy(nd_class); |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 1248 | err_class: |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1249 | unregister_chrdev(nvdimm_major, "dimmctl"); |
| 1250 | err_dimm_chrdev: |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 1251 | unregister_chrdev(nvdimm_bus_major, "ndctl"); |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1252 | err_bus_chrdev: |
Dan Williams | e6dfb2d | 2015-04-25 03:56:17 -0400 | [diff] [blame] | 1253 | bus_unregister(&nvdimm_bus_type); |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 1254 | |
| 1255 | return rc; |
| 1256 | } |
| 1257 | |
Dan Williams | 4d88a97 | 2015-05-31 14:41:48 -0400 | [diff] [blame] | 1258 | void nvdimm_bus_exit(void) |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 1259 | { |
Dan Williams | 1851594 | 2016-07-22 23:46:08 -0700 | [diff] [blame] | 1260 | driver_unregister(&nd_bus_driver.drv); |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 1261 | class_destroy(nd_class); |
| 1262 | unregister_chrdev(nvdimm_bus_major, "ndctl"); |
Dan Williams | 62232e45 | 2015-06-08 14:27:06 -0400 | [diff] [blame] | 1263 | unregister_chrdev(nvdimm_major, "dimmctl"); |
Dan Williams | e6dfb2d | 2015-04-25 03:56:17 -0400 | [diff] [blame] | 1264 | bus_unregister(&nvdimm_bus_type); |
Dan Williams | 1851594 | 2016-07-22 23:46:08 -0700 | [diff] [blame] | 1265 | ida_destroy(&nd_ida); |
Dan Williams | 45def22 | 2015-04-26 19:26:48 -0400 | [diff] [blame] | 1266 | } |