blob: 571a6c7ee2fc05b38d056df1423a9d741f53c5ac [file] [log] [blame]
Ross Zwisler9e853f22015-04-01 09:12:19 +02001/*
2 * Persistent Memory Driver
3 *
Dan Williams9f53f9f2015-06-09 15:33:45 -04004 * Copyright (c) 2014-2015, Intel Corporation.
Ross Zwisler9e853f22015-04-01 09:12:19 +02005 * Copyright (c) 2015, Christoph Hellwig <hch@lst.de>.
6 * Copyright (c) 2015, Boaz Harrosh <boaz@plexistor.com>.
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms and conditions of the GNU General Public License,
10 * version 2, as published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope it will be useful, but WITHOUT
13 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 * more details.
16 */
17
18#include <asm/cacheflush.h>
19#include <linux/blkdev.h>
20#include <linux/hdreg.h>
21#include <linux/init.h>
22#include <linux/platform_device.h>
23#include <linux/module.h>
24#include <linux/moduleparam.h>
Dan Williamsb95f5f42016-01-04 23:50:23 -080025#include <linux/badblocks.h>
Dan Williams9476df72016-01-15 16:56:19 -080026#include <linux/memremap.h>
Dan Williams32ab0a3f2015-08-01 02:16:37 -040027#include <linux/vmalloc.h>
Dan Williams34c0fd52016-01-15 16:56:14 -080028#include <linux/pfn_t.h>
Ross Zwisler9e853f22015-04-01 09:12:19 +020029#include <linux/slab.h>
Ross Zwisler61031952015-06-25 03:08:39 -040030#include <linux/pmem.h>
Dan Williams9f53f9f2015-06-09 15:33:45 -040031#include <linux/nd.h>
Dan Williamsf295e532016-06-17 11:08:06 -070032#include "pmem.h"
Dan Williams32ab0a3f2015-08-01 02:16:37 -040033#include "pfn.h"
Dan Williams9f53f9f2015-06-09 15:33:45 -040034#include "nd.h"
Ross Zwisler9e853f22015-04-01 09:12:19 +020035
Dan Williamsf284a4f2016-07-07 19:44:50 -070036static struct device *to_dev(struct pmem_device *pmem)
37{
38 /*
39 * nvdimm bus services need a 'dev' parameter, and we record the device
40 * at init in bb.dev.
41 */
42 return pmem->bb.dev;
43}
44
45static struct nd_region *to_region(struct pmem_device *pmem)
46{
47 return to_nd_region(to_dev(pmem)->parent);
48}
Ross Zwisler9e853f22015-04-01 09:12:19 +020049
Dan Williams59e64732016-03-08 07:16:07 -080050static void pmem_clear_poison(struct pmem_device *pmem, phys_addr_t offset,
51 unsigned int len)
52{
Dan Williamsf284a4f2016-07-07 19:44:50 -070053 struct device *dev = to_dev(pmem);
Dan Williams59e64732016-03-08 07:16:07 -080054 sector_t sector;
55 long cleared;
56
57 sector = (offset - pmem->data_offset) / 512;
58 cleared = nvdimm_clear_poison(dev, pmem->phys_addr + offset, len);
59
60 if (cleared > 0 && cleared / 512) {
Vishal Verma5bf0b6e2016-07-22 17:21:31 -060061 dev_dbg(dev, "%s: %#llx clear %ld sector%s\n",
Dan Williams59e64732016-03-08 07:16:07 -080062 __func__, (unsigned long long) sector,
63 cleared / 512, cleared / 512 > 1 ? "s" : "");
64 badblocks_clear(&pmem->bb, sector, cleared / 512);
65 }
66 invalidate_pmem(pmem->virt_addr + offset, len);
67}
68
Dan Williamse10624f2016-01-06 12:03:41 -080069static int pmem_do_bvec(struct pmem_device *pmem, struct page *page,
Jens Axboec11f0c02016-08-05 08:11:04 -060070 unsigned int len, unsigned int off, bool is_write,
Ross Zwisler9e853f22015-04-01 09:12:19 +020071 sector_t sector)
72{
Dan Williamsb5ebc8e2016-03-06 15:20:51 -080073 int rc = 0;
Dan Williams59e64732016-03-08 07:16:07 -080074 bool bad_pmem = false;
Ross Zwisler9e853f22015-04-01 09:12:19 +020075 void *mem = kmap_atomic(page);
Dan Williams32ab0a3f2015-08-01 02:16:37 -040076 phys_addr_t pmem_off = sector * 512 + pmem->data_offset;
Dan Williams7a9eb202016-06-03 18:06:47 -070077 void *pmem_addr = pmem->virt_addr + pmem_off;
Ross Zwisler9e853f22015-04-01 09:12:19 +020078
Dan Williams59e64732016-03-08 07:16:07 -080079 if (unlikely(is_bad_pmem(&pmem->bb, sector, len)))
80 bad_pmem = true;
81
Jens Axboec11f0c02016-08-05 08:11:04 -060082 if (!is_write) {
Dan Williams59e64732016-03-08 07:16:07 -080083 if (unlikely(bad_pmem))
Dan Williamsb5ebc8e2016-03-06 15:20:51 -080084 rc = -EIO;
85 else {
Dan Williamsfc0c2022016-03-08 10:30:19 -080086 rc = memcpy_from_pmem(mem + off, pmem_addr, len);
Dan Williamsb5ebc8e2016-03-06 15:20:51 -080087 flush_dcache_page(page);
88 }
Ross Zwisler9e853f22015-04-01 09:12:19 +020089 } else {
Dan Williams0a370d262016-04-14 19:40:47 -070090 /*
91 * Note that we write the data both before and after
92 * clearing poison. The write before clear poison
93 * handles situations where the latest written data is
94 * preserved and the clear poison operation simply marks
95 * the address range as valid without changing the data.
96 * In this case application software can assume that an
97 * interrupted write will either return the new good
98 * data or an error.
99 *
100 * However, if pmem_clear_poison() leaves the data in an
101 * indeterminate state we need to perform the write
102 * after clear poison.
103 */
Ross Zwisler9e853f22015-04-01 09:12:19 +0200104 flush_dcache_page(page);
Ross Zwisler61031952015-06-25 03:08:39 -0400105 memcpy_to_pmem(pmem_addr, mem + off, len);
Dan Williams59e64732016-03-08 07:16:07 -0800106 if (unlikely(bad_pmem)) {
107 pmem_clear_poison(pmem, pmem_off, len);
108 memcpy_to_pmem(pmem_addr, mem + off, len);
109 }
Ross Zwisler9e853f22015-04-01 09:12:19 +0200110 }
111
112 kunmap_atomic(mem);
Dan Williamsb5ebc8e2016-03-06 15:20:51 -0800113 return rc;
Ross Zwisler9e853f22015-04-01 09:12:19 +0200114}
115
Dan Williams7e267a82016-06-01 20:48:15 -0700116/* account for REQ_FLUSH rename, replace with REQ_PREFLUSH after v4.8-rc1 */
117#ifndef REQ_FLUSH
118#define REQ_FLUSH REQ_PREFLUSH
119#endif
120
Jens Axboedece1632015-11-05 10:41:16 -0700121static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio)
Ross Zwisler9e853f22015-04-01 09:12:19 +0200122{
Dan Williamse10624f2016-01-06 12:03:41 -0800123 int rc = 0;
Dan Williamsf0dc0892015-05-16 12:28:53 -0400124 bool do_acct;
125 unsigned long start;
Dan Williamsedc870e2015-05-16 12:28:51 -0400126 struct bio_vec bvec;
127 struct bvec_iter iter;
Dan Williamsbd842b82016-03-18 23:47:43 -0700128 struct pmem_device *pmem = q->queuedata;
Dan Williams7e267a82016-06-01 20:48:15 -0700129 struct nd_region *nd_region = to_region(pmem);
130
Jens Axboe1eff9d32016-08-05 15:35:16 -0600131 if (bio->bi_opf & REQ_FLUSH)
Dan Williams7e267a82016-06-01 20:48:15 -0700132 nvdimm_flush(nd_region);
Ross Zwisler9e853f22015-04-01 09:12:19 +0200133
Dan Williamsf0dc0892015-05-16 12:28:53 -0400134 do_acct = nd_iostat_start(bio, &start);
Dan Williamse10624f2016-01-06 12:03:41 -0800135 bio_for_each_segment(bvec, bio, iter) {
136 rc = pmem_do_bvec(pmem, bvec.bv_page, bvec.bv_len,
Jens Axboec11f0c02016-08-05 08:11:04 -0600137 bvec.bv_offset, op_is_write(bio_op(bio)),
Dan Williamse10624f2016-01-06 12:03:41 -0800138 iter.bi_sector);
139 if (rc) {
140 bio->bi_error = rc;
141 break;
142 }
143 }
Dan Williamsf0dc0892015-05-16 12:28:53 -0400144 if (do_acct)
145 nd_iostat_end(bio, start);
Ross Zwisler61031952015-06-25 03:08:39 -0400146
Jens Axboe1eff9d32016-08-05 15:35:16 -0600147 if (bio->bi_opf & REQ_FUA)
Dan Williams7e267a82016-06-01 20:48:15 -0700148 nvdimm_flush(nd_region);
Ross Zwisler61031952015-06-25 03:08:39 -0400149
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200150 bio_endio(bio);
Jens Axboedece1632015-11-05 10:41:16 -0700151 return BLK_QC_T_NONE;
Ross Zwisler9e853f22015-04-01 09:12:19 +0200152}
153
154static int pmem_rw_page(struct block_device *bdev, sector_t sector,
Jens Axboec11f0c02016-08-05 08:11:04 -0600155 struct page *page, bool is_write)
Ross Zwisler9e853f22015-04-01 09:12:19 +0200156{
Dan Williamsbd842b82016-03-18 23:47:43 -0700157 struct pmem_device *pmem = bdev->bd_queue->queuedata;
Dan Williamse10624f2016-01-06 12:03:41 -0800158 int rc;
Ross Zwisler9e853f22015-04-01 09:12:19 +0200159
Jens Axboec11f0c02016-08-05 08:11:04 -0600160 rc = pmem_do_bvec(pmem, page, PAGE_SIZE, 0, is_write, sector);
Ross Zwisler9e853f22015-04-01 09:12:19 +0200161
Dan Williamse10624f2016-01-06 12:03:41 -0800162 /*
163 * The ->rw_page interface is subtle and tricky. The core
164 * retries on any error, so we can only invoke page_endio() in
165 * the successful completion case. Otherwise, we'll see crashes
166 * caused by double completion.
167 */
168 if (rc == 0)
Jens Axboec11f0c02016-08-05 08:11:04 -0600169 page_endio(page, is_write, 0);
Dan Williamse10624f2016-01-06 12:03:41 -0800170
171 return rc;
Ross Zwisler9e853f22015-04-01 09:12:19 +0200172}
173
Dan Williamsf295e532016-06-17 11:08:06 -0700174/* see "strong" declaration in tools/testing/nvdimm/pmem-dax.c */
175__weak long pmem_direct_access(struct block_device *bdev, sector_t sector,
Dan Williams7a9eb202016-06-03 18:06:47 -0700176 void **kaddr, pfn_t *pfn, long size)
Ross Zwisler9e853f22015-04-01 09:12:19 +0200177{
Dan Williamsbd842b82016-03-18 23:47:43 -0700178 struct pmem_device *pmem = bdev->bd_queue->queuedata;
Dan Williams32ab0a3f2015-08-01 02:16:37 -0400179 resource_size_t offset = sector * 512 + pmem->data_offset;
Ross Zwisler9e853f22015-04-01 09:12:19 +0200180
Dan Williams0a70bd42016-02-24 14:02:11 -0800181 if (unlikely(is_bad_pmem(&pmem->bb, sector, size)))
182 return -EIO;
Ross Zwislere2e05392015-08-18 13:55:41 -0600183 *kaddr = pmem->virt_addr + offset;
Dan Williams34c0fd52016-01-15 16:56:14 -0800184 *pfn = phys_to_pfn_t(pmem->phys_addr + offset, pmem->pfn_flags);
Ross Zwisler9e853f22015-04-01 09:12:19 +0200185
Dan Williams0a70bd42016-02-24 14:02:11 -0800186 /*
187 * If badblocks are present, limit known good range to the
188 * requested range.
189 */
190 if (unlikely(pmem->bb.count))
191 return size;
Dan Williamscfe30b82016-03-03 09:38:00 -0800192 return pmem->size - pmem->pfn_pad - offset;
Ross Zwisler9e853f22015-04-01 09:12:19 +0200193}
194
195static const struct block_device_operations pmem_fops = {
196 .owner = THIS_MODULE,
197 .rw_page = pmem_rw_page,
198 .direct_access = pmem_direct_access,
Dan Williams58138822015-06-23 20:08:34 -0400199 .revalidate_disk = nvdimm_revalidate_disk,
Ross Zwisler9e853f22015-04-01 09:12:19 +0200200};
201
Dan Williams030b99e2016-03-17 20:24:31 -0700202static void pmem_release_queue(void *q)
203{
204 blk_cleanup_queue(q);
205}
206
Dan Williamsf02716d2016-06-15 14:59:17 -0700207static void pmem_release_disk(void *disk)
Dan Williams030b99e2016-03-17 20:24:31 -0700208{
209 del_gendisk(disk);
210 put_disk(disk);
211}
212
Dan Williams200c79d2016-03-22 00:22:16 -0700213static int pmem_attach_disk(struct device *dev,
214 struct nd_namespace_common *ndns)
Ross Zwisler9e853f22015-04-01 09:12:19 +0200215{
Dan Williams200c79d2016-03-22 00:22:16 -0700216 struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev);
Dan Williamsf284a4f2016-07-07 19:44:50 -0700217 struct nd_region *nd_region = to_nd_region(dev->parent);
Dan Williams200c79d2016-03-22 00:22:16 -0700218 struct vmem_altmap __altmap, *altmap = NULL;
219 struct resource *res = &nsio->res;
220 struct nd_pfn *nd_pfn = NULL;
221 int nid = dev_to_node(dev);
222 struct nd_pfn_sb *pfn_sb;
Ross Zwisler9e853f22015-04-01 09:12:19 +0200223 struct pmem_device *pmem;
Dan Williams200c79d2016-03-22 00:22:16 -0700224 struct resource pfn_res;
Dan Williams468ded02016-01-15 16:56:46 -0800225 struct request_queue *q;
Dan Williams200c79d2016-03-22 00:22:16 -0700226 struct gendisk *disk;
227 void *addr;
228
229 /* while nsio_rw_bytes is active, parse a pfn info block if present */
230 if (is_nd_pfn(dev)) {
231 nd_pfn = to_nd_pfn(dev);
232 altmap = nvdimm_setup_pfn(nd_pfn, &pfn_res, &__altmap);
233 if (IS_ERR(altmap))
234 return PTR_ERR(altmap);
235 }
236
237 /* we're attaching a block device, disable raw namespace access */
238 devm_nsio_disable(dev, nsio);
Ross Zwisler9e853f22015-04-01 09:12:19 +0200239
Christoph Hellwig708ab622015-08-10 23:07:08 -0400240 pmem = devm_kzalloc(dev, sizeof(*pmem), GFP_KERNEL);
Ross Zwisler9e853f22015-04-01 09:12:19 +0200241 if (!pmem)
Dan Williams200c79d2016-03-22 00:22:16 -0700242 return -ENOMEM;
Ross Zwisler9e853f22015-04-01 09:12:19 +0200243
Dan Williams200c79d2016-03-22 00:22:16 -0700244 dev_set_drvdata(dev, pmem);
Ross Zwisler9e853f22015-04-01 09:12:19 +0200245 pmem->phys_addr = res->start;
246 pmem->size = resource_size(res);
Dan Williamsf284a4f2016-07-07 19:44:50 -0700247 if (nvdimm_has_flush(nd_region) < 0)
Ross Zwisler61031952015-06-25 03:08:39 -0400248 dev_warn(dev, "unable to guarantee persistence of writes\n");
Ross Zwisler9e853f22015-04-01 09:12:19 +0200249
Dan Williams947df022016-03-21 22:28:40 -0700250 if (!devm_request_mem_region(dev, res->start, resource_size(res),
251 dev_name(dev))) {
252 dev_warn(dev, "could not reserve region %pR\n", res);
Dan Williams200c79d2016-03-22 00:22:16 -0700253 return -EBUSY;
Ross Zwisler9e853f22015-04-01 09:12:19 +0200254 }
255
Dan Williams468ded02016-01-15 16:56:46 -0800256 q = blk_alloc_queue_node(GFP_KERNEL, dev_to_node(dev));
257 if (!q)
Dan Williams200c79d2016-03-22 00:22:16 -0700258 return -ENOMEM;
Dan Williams468ded02016-01-15 16:56:46 -0800259
Dan Williams34c0fd52016-01-15 16:56:14 -0800260 pmem->pfn_flags = PFN_DEV;
Dan Williams200c79d2016-03-22 00:22:16 -0700261 if (is_nd_pfn(dev)) {
262 addr = devm_memremap_pages(dev, &pfn_res, &q->q_usage_counter,
263 altmap);
264 pfn_sb = nd_pfn->pfn_sb;
265 pmem->data_offset = le64_to_cpu(pfn_sb->dataoff);
266 pmem->pfn_pad = resource_size(res) - resource_size(&pfn_res);
267 pmem->pfn_flags |= PFN_MAP;
268 res = &pfn_res; /* for badblocks populate */
269 res->start += pmem->data_offset;
270 } else if (pmem_should_map_pages(dev)) {
271 addr = devm_memremap_pages(dev, &nsio->res,
Dan Williams5c2c2582016-01-15 16:56:49 -0800272 &q->q_usage_counter, NULL);
Dan Williams34c0fd52016-01-15 16:56:14 -0800273 pmem->pfn_flags |= PFN_MAP;
274 } else
Dan Williams200c79d2016-03-22 00:22:16 -0700275 addr = devm_memremap(dev, pmem->phys_addr,
276 pmem->size, ARCH_MEMREMAP_PMEM);
Dan Williamsb36f4762015-09-15 02:42:20 -0400277
Dan Williams030b99e2016-03-17 20:24:31 -0700278 /*
279 * At release time the queue must be dead before
280 * devm_memremap_pages is unwound
281 */
Dan Williamsf02716d2016-06-15 14:59:17 -0700282 if (devm_add_action_or_reset(dev, pmem_release_queue, q))
Dan Williams200c79d2016-03-22 00:22:16 -0700283 return -ENOMEM;
Dan Williams8c2f7e82015-06-25 04:20:04 -0400284
Dan Williams200c79d2016-03-22 00:22:16 -0700285 if (IS_ERR(addr))
286 return PTR_ERR(addr);
Dan Williams7a9eb202016-06-03 18:06:47 -0700287 pmem->virt_addr = addr;
Ross Zwisler9e853f22015-04-01 09:12:19 +0200288
Dan Williams7e267a82016-06-01 20:48:15 -0700289 blk_queue_write_cache(q, true, true);
Dan Williams5a922892016-03-21 15:43:53 -0700290 blk_queue_make_request(q, pmem_make_request);
291 blk_queue_physical_block_size(q, PAGE_SIZE);
292 blk_queue_max_hw_sectors(q, UINT_MAX);
293 blk_queue_bounce_limit(q, BLK_BOUNCE_ANY);
294 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q);
Toshi Kani163d4ba2016-06-23 17:05:50 -0400295 queue_flag_set_unlocked(QUEUE_FLAG_DAX, q);
Dan Williams5a922892016-03-21 15:43:53 -0700296 q->queuedata = pmem;
Ross Zwisler9e853f22015-04-01 09:12:19 +0200297
Dan Williams538ea4a2015-10-05 20:35:56 -0400298 disk = alloc_disk_node(0, nid);
Dan Williams030b99e2016-03-17 20:24:31 -0700299 if (!disk)
300 return -ENOMEM;
Ross Zwisler9e853f22015-04-01 09:12:19 +0200301
Ross Zwisler9e853f22015-04-01 09:12:19 +0200302 disk->fops = &pmem_fops;
Dan Williams5a922892016-03-21 15:43:53 -0700303 disk->queue = q;
Ross Zwisler9e853f22015-04-01 09:12:19 +0200304 disk->flags = GENHD_FL_EXT_DEVT;
Vishal Verma5212e112015-06-25 04:20:32 -0400305 nvdimm_namespace_disk_name(ndns, disk->disk_name);
Dan Williamscfe30b82016-03-03 09:38:00 -0800306 set_capacity(disk, (pmem->size - pmem->pfn_pad - pmem->data_offset)
307 / 512);
Dan Williamsb95f5f42016-01-04 23:50:23 -0800308 if (devm_init_badblocks(dev, &pmem->bb))
309 return -ENOMEM;
Dan Williamsf284a4f2016-07-07 19:44:50 -0700310 nvdimm_badblocks_populate(nd_region, &pmem->bb, res);
Dan Williams57f7f312016-01-06 12:03:42 -0800311 disk->bb = &pmem->bb;
Dan Williams0d52c7562016-06-15 19:44:20 -0700312 device_add_disk(dev, disk);
Dan Williamsf02716d2016-06-15 14:59:17 -0700313
314 if (devm_add_action_or_reset(dev, pmem_release_disk, disk))
315 return -ENOMEM;
316
Dan Williams58138822015-06-23 20:08:34 -0400317 revalidate_disk(disk);
Ross Zwisler9e853f22015-04-01 09:12:19 +0200318
Dan Williams8c2f7e82015-06-25 04:20:04 -0400319 return 0;
320}
Ross Zwisler9e853f22015-04-01 09:12:19 +0200321
Dan Williams9f53f9f2015-06-09 15:33:45 -0400322static int nd_pmem_probe(struct device *dev)
Ross Zwisler9e853f22015-04-01 09:12:19 +0200323{
Dan Williams8c2f7e82015-06-25 04:20:04 -0400324 struct nd_namespace_common *ndns;
Ross Zwisler9e853f22015-04-01 09:12:19 +0200325
Dan Williams8c2f7e82015-06-25 04:20:04 -0400326 ndns = nvdimm_namespace_common_probe(dev);
327 if (IS_ERR(ndns))
328 return PTR_ERR(ndns);
Dan Williamsbf9bccc2015-06-17 17:14:46 -0400329
Dan Williams200c79d2016-03-22 00:22:16 -0700330 if (devm_nsio_enable(dev, to_nd_namespace_io(&ndns->dev)))
331 return -ENXIO;
Ross Zwisler9e853f22015-04-01 09:12:19 +0200332
Dan Williams200c79d2016-03-22 00:22:16 -0700333 if (is_nd_btt(dev))
Christoph Hellwig708ab622015-08-10 23:07:08 -0400334 return nvdimm_namespace_attach_btt(ndns);
335
Dan Williams32ab0a3f2015-08-01 02:16:37 -0400336 if (is_nd_pfn(dev))
Dan Williams200c79d2016-03-22 00:22:16 -0700337 return pmem_attach_disk(dev, ndns);
Dan Williams32ab0a3f2015-08-01 02:16:37 -0400338
Dan Williams200c79d2016-03-22 00:22:16 -0700339 /* if we find a valid info-block we'll come back as that personality */
Dan Williamsc5ed9262016-05-18 14:50:12 -0700340 if (nd_btt_probe(dev, ndns) == 0 || nd_pfn_probe(dev, ndns) == 0
341 || nd_dax_probe(dev, ndns) == 0)
Dan Williams32ab0a3f2015-08-01 02:16:37 -0400342 return -ENXIO;
Dan Williams32ab0a3f2015-08-01 02:16:37 -0400343
Dan Williams200c79d2016-03-22 00:22:16 -0700344 /* ...otherwise we're just a raw pmem device */
345 return pmem_attach_disk(dev, ndns);
Ross Zwisler9e853f22015-04-01 09:12:19 +0200346}
347
Dan Williams9f53f9f2015-06-09 15:33:45 -0400348static int nd_pmem_remove(struct device *dev)
Ross Zwisler9e853f22015-04-01 09:12:19 +0200349{
Dan Williams8c2f7e82015-06-25 04:20:04 -0400350 if (is_nd_btt(dev))
Dan Williams298f2bc2016-03-15 16:41:04 -0700351 nvdimm_namespace_detach_btt(to_nd_btt(dev));
Dan Williams476f8482016-07-09 00:12:52 -0700352 nvdimm_flush(to_nd_region(dev->parent));
353
Ross Zwisler9e853f22015-04-01 09:12:19 +0200354 return 0;
355}
356
Dan Williams476f8482016-07-09 00:12:52 -0700357static void nd_pmem_shutdown(struct device *dev)
358{
359 nvdimm_flush(to_nd_region(dev->parent));
360}
361
Dan Williams71999462016-02-18 10:29:49 -0800362static void nd_pmem_notify(struct device *dev, enum nvdimm_event event)
363{
Dan Williams298f2bc2016-03-15 16:41:04 -0700364 struct pmem_device *pmem = dev_get_drvdata(dev);
Dan Williamsf284a4f2016-07-07 19:44:50 -0700365 struct nd_region *nd_region = to_region(pmem);
Dan Williams298f2bc2016-03-15 16:41:04 -0700366 resource_size_t offset = 0, end_trunc = 0;
367 struct nd_namespace_common *ndns;
368 struct nd_namespace_io *nsio;
369 struct resource res;
Dan Williams71999462016-02-18 10:29:49 -0800370
371 if (event != NVDIMM_REVALIDATE_POISON)
372 return;
373
Dan Williams298f2bc2016-03-15 16:41:04 -0700374 if (is_nd_btt(dev)) {
375 struct nd_btt *nd_btt = to_nd_btt(dev);
376
377 ndns = nd_btt->ndns;
378 } else if (is_nd_pfn(dev)) {
Dan Williamsa3901802016-04-07 20:02:06 -0700379 struct nd_pfn *nd_pfn = to_nd_pfn(dev);
380 struct nd_pfn_sb *pfn_sb = nd_pfn->pfn_sb;
381
Dan Williams298f2bc2016-03-15 16:41:04 -0700382 ndns = nd_pfn->ndns;
383 offset = pmem->data_offset + __le32_to_cpu(pfn_sb->start_pad);
384 end_trunc = __le32_to_cpu(pfn_sb->end_trunc);
385 } else
386 ndns = to_ndns(dev);
Dan Williamsa3901802016-04-07 20:02:06 -0700387
Dan Williams298f2bc2016-03-15 16:41:04 -0700388 nsio = to_nd_namespace_io(&ndns->dev);
389 res.start = nsio->res.start + offset;
390 res.end = nsio->res.end - end_trunc;
Dan Williamsa3901802016-04-07 20:02:06 -0700391 nvdimm_badblocks_populate(nd_region, &pmem->bb, &res);
Dan Williams71999462016-02-18 10:29:49 -0800392}
393
Dan Williams9f53f9f2015-06-09 15:33:45 -0400394MODULE_ALIAS("pmem");
395MODULE_ALIAS_ND_DEVICE(ND_DEVICE_NAMESPACE_IO);
Dan Williamsbf9bccc2015-06-17 17:14:46 -0400396MODULE_ALIAS_ND_DEVICE(ND_DEVICE_NAMESPACE_PMEM);
Dan Williams9f53f9f2015-06-09 15:33:45 -0400397static struct nd_device_driver nd_pmem_driver = {
398 .probe = nd_pmem_probe,
399 .remove = nd_pmem_remove,
Dan Williams71999462016-02-18 10:29:49 -0800400 .notify = nd_pmem_notify,
Dan Williams476f8482016-07-09 00:12:52 -0700401 .shutdown = nd_pmem_shutdown,
Dan Williams9f53f9f2015-06-09 15:33:45 -0400402 .drv = {
403 .name = "nd_pmem",
Ross Zwisler9e853f22015-04-01 09:12:19 +0200404 },
Dan Williamsbf9bccc2015-06-17 17:14:46 -0400405 .type = ND_DRIVER_NAMESPACE_IO | ND_DRIVER_NAMESPACE_PMEM,
Ross Zwisler9e853f22015-04-01 09:12:19 +0200406};
407
408static int __init pmem_init(void)
409{
NeilBrown55155292016-03-09 09:21:54 +1100410 return nd_driver_register(&nd_pmem_driver);
Ross Zwisler9e853f22015-04-01 09:12:19 +0200411}
412module_init(pmem_init);
413
414static void pmem_exit(void)
415{
Dan Williams9f53f9f2015-06-09 15:33:45 -0400416 driver_unregister(&nd_pmem_driver.drv);
Ross Zwisler9e853f22015-04-01 09:12:19 +0200417}
418module_exit(pmem_exit);
419
420MODULE_AUTHOR("Ross Zwisler <ross.zwisler@linux.intel.com>");
421MODULE_LICENSE("GPL v2");