Benny Halevy | c93407d | 2011-05-22 19:49:06 +0300 | [diff] [blame] | 1 | /* |
| 2 | * pNFS Objects layout implementation over open-osd initiator library |
| 3 | * |
| 4 | * Copyright (C) 2009 Panasas Inc. [year of first publication] |
| 5 | * All rights reserved. |
| 6 | * |
| 7 | * Benny Halevy <bhalevy@panasas.com> |
| 8 | * Boaz Harrosh <bharrosh@panasas.com> |
| 9 | * |
| 10 | * This program is free software; you can redistribute it and/or modify |
| 11 | * it under the terms of the GNU General Public License version 2 |
| 12 | * See the file COPYING included with this distribution for more details. |
| 13 | * |
| 14 | * Redistribution and use in source and binary forms, with or without |
| 15 | * modification, are permitted provided that the following conditions |
| 16 | * are met: |
| 17 | * |
| 18 | * 1. Redistributions of source code must retain the above copyright |
| 19 | * notice, this list of conditions and the following disclaimer. |
| 20 | * 2. Redistributions in binary form must reproduce the above copyright |
| 21 | * notice, this list of conditions and the following disclaimer in the |
| 22 | * documentation and/or other materials provided with the distribution. |
| 23 | * 3. Neither the name of the Panasas company nor the names of its |
| 24 | * contributors may be used to endorse or promote products derived |
| 25 | * from this software without specific prior written permission. |
| 26 | * |
| 27 | * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED |
| 28 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF |
| 29 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| 30 | * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE |
| 31 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| 32 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| 33 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR |
| 34 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| 35 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| 36 | * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| 37 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 38 | */ |
| 39 | |
| 40 | #include <linux/module.h> |
Boaz Harrosh | 09f5bf4 | 2011-05-22 19:50:20 +0300 | [diff] [blame] | 41 | #include <scsi/osd_initiator.h> |
| 42 | |
| 43 | #include "objlayout.h" |
| 44 | |
| 45 | #define NFSDBG_FACILITY NFSDBG_PNFS_LD |
| 46 | |
| 47 | #define _LLU(x) ((unsigned long long)x) |
| 48 | |
Boaz Harrosh | b6c05f1 | 2011-05-26 21:45:34 +0300 | [diff] [blame] | 49 | struct objio_dev_ent { |
| 50 | struct nfs4_deviceid_node id_node; |
| 51 | struct osd_dev *od; |
| 52 | }; |
| 53 | |
| 54 | static void |
| 55 | objio_free_deviceid_node(struct nfs4_deviceid_node *d) |
| 56 | { |
| 57 | struct objio_dev_ent *de = container_of(d, struct objio_dev_ent, id_node); |
| 58 | |
| 59 | dprintk("%s: free od=%p\n", __func__, de->od); |
| 60 | osduld_put_device(de->od); |
| 61 | kfree(de); |
| 62 | } |
| 63 | |
| 64 | static struct objio_dev_ent *_dev_list_find(const struct nfs_server *nfss, |
| 65 | const struct nfs4_deviceid *d_id) |
| 66 | { |
| 67 | struct nfs4_deviceid_node *d; |
| 68 | struct objio_dev_ent *de; |
| 69 | |
| 70 | d = nfs4_find_get_deviceid(nfss->pnfs_curr_ld, nfss->nfs_client, d_id); |
| 71 | if (!d) |
| 72 | return NULL; |
| 73 | |
| 74 | de = container_of(d, struct objio_dev_ent, id_node); |
| 75 | return de; |
| 76 | } |
| 77 | |
| 78 | static struct objio_dev_ent * |
| 79 | _dev_list_add(const struct nfs_server *nfss, |
| 80 | const struct nfs4_deviceid *d_id, struct osd_dev *od, |
| 81 | gfp_t gfp_flags) |
| 82 | { |
| 83 | struct nfs4_deviceid_node *d; |
| 84 | struct objio_dev_ent *de = kzalloc(sizeof(*de), gfp_flags); |
| 85 | struct objio_dev_ent *n; |
| 86 | |
| 87 | if (!de) { |
| 88 | dprintk("%s: -ENOMEM od=%p\n", __func__, od); |
| 89 | return NULL; |
| 90 | } |
| 91 | |
| 92 | dprintk("%s: Adding od=%p\n", __func__, od); |
| 93 | nfs4_init_deviceid_node(&de->id_node, |
| 94 | nfss->pnfs_curr_ld, |
| 95 | nfss->nfs_client, |
| 96 | d_id); |
| 97 | de->od = od; |
| 98 | |
| 99 | d = nfs4_insert_deviceid_node(&de->id_node); |
| 100 | n = container_of(d, struct objio_dev_ent, id_node); |
| 101 | if (n != de) { |
| 102 | dprintk("%s: Race with other n->od=%p\n", __func__, n->od); |
| 103 | objio_free_deviceid_node(&de->id_node); |
| 104 | de = n; |
| 105 | } |
| 106 | |
| 107 | atomic_inc(&de->id_node.ref); |
| 108 | return de; |
| 109 | } |
| 110 | |
Boaz Harrosh | 09f5bf4 | 2011-05-22 19:50:20 +0300 | [diff] [blame] | 111 | struct caps_buffers { |
| 112 | u8 caps_key[OSD_CRYPTO_KEYID_SIZE]; |
| 113 | u8 creds[OSD_CAP_LEN]; |
| 114 | }; |
| 115 | |
| 116 | struct objio_segment { |
| 117 | struct pnfs_layout_segment lseg; |
| 118 | |
| 119 | struct pnfs_osd_object_cred *comps; |
| 120 | |
| 121 | unsigned mirrors_p1; |
| 122 | unsigned stripe_unit; |
| 123 | unsigned group_width; /* Data stripe_units without integrity comps */ |
| 124 | u64 group_depth; |
| 125 | unsigned group_count; |
| 126 | |
| 127 | unsigned comps_index; |
| 128 | unsigned num_comps; |
| 129 | /* variable length */ |
| 130 | struct objio_dev_ent *ods[]; |
| 131 | }; |
| 132 | |
| 133 | static inline struct objio_segment * |
| 134 | OBJIO_LSEG(struct pnfs_layout_segment *lseg) |
| 135 | { |
| 136 | return container_of(lseg, struct objio_segment, lseg); |
| 137 | } |
| 138 | |
Boaz Harrosh | b6c05f1 | 2011-05-26 21:45:34 +0300 | [diff] [blame] | 139 | /* Send and wait for a get_device_info of devices in the layout, |
| 140 | then look them up with the osd_initiator library */ |
| 141 | static struct objio_dev_ent *_device_lookup(struct pnfs_layout_hdr *pnfslay, |
| 142 | struct objio_segment *objio_seg, unsigned comp, |
| 143 | gfp_t gfp_flags) |
| 144 | { |
| 145 | struct pnfs_osd_deviceaddr *deviceaddr; |
| 146 | struct nfs4_deviceid *d_id; |
| 147 | struct objio_dev_ent *ode; |
| 148 | struct osd_dev *od; |
| 149 | struct osd_dev_info odi; |
| 150 | int err; |
| 151 | |
| 152 | d_id = &objio_seg->comps[comp].oc_object_id.oid_device_id; |
| 153 | |
| 154 | ode = _dev_list_find(NFS_SERVER(pnfslay->plh_inode), d_id); |
| 155 | if (ode) |
| 156 | return ode; |
| 157 | |
| 158 | err = objlayout_get_deviceinfo(pnfslay, d_id, &deviceaddr, gfp_flags); |
| 159 | if (unlikely(err)) { |
| 160 | dprintk("%s: objlayout_get_deviceinfo dev(%llx:%llx) =>%d\n", |
| 161 | __func__, _DEVID_LO(d_id), _DEVID_HI(d_id), err); |
| 162 | return ERR_PTR(err); |
| 163 | } |
| 164 | |
| 165 | odi.systemid_len = deviceaddr->oda_systemid.len; |
| 166 | if (odi.systemid_len > sizeof(odi.systemid)) { |
| 167 | err = -EINVAL; |
| 168 | goto out; |
| 169 | } else if (odi.systemid_len) |
| 170 | memcpy(odi.systemid, deviceaddr->oda_systemid.data, |
| 171 | odi.systemid_len); |
| 172 | odi.osdname_len = deviceaddr->oda_osdname.len; |
| 173 | odi.osdname = (u8 *)deviceaddr->oda_osdname.data; |
| 174 | |
| 175 | if (!odi.osdname_len && !odi.systemid_len) { |
| 176 | dprintk("%s: !odi.osdname_len && !odi.systemid_len\n", |
| 177 | __func__); |
| 178 | err = -ENODEV; |
| 179 | goto out; |
| 180 | } |
| 181 | |
| 182 | od = osduld_info_lookup(&odi); |
| 183 | if (unlikely(IS_ERR(od))) { |
| 184 | err = PTR_ERR(od); |
| 185 | dprintk("%s: osduld_info_lookup => %d\n", __func__, err); |
| 186 | goto out; |
| 187 | } |
| 188 | |
| 189 | ode = _dev_list_add(NFS_SERVER(pnfslay->plh_inode), d_id, od, |
| 190 | gfp_flags); |
| 191 | |
| 192 | out: |
| 193 | dprintk("%s: return=%d\n", __func__, err); |
| 194 | objlayout_put_deviceinfo(deviceaddr); |
| 195 | return err ? ERR_PTR(err) : ode; |
| 196 | } |
| 197 | |
| 198 | static int objio_devices_lookup(struct pnfs_layout_hdr *pnfslay, |
| 199 | struct objio_segment *objio_seg, |
| 200 | gfp_t gfp_flags) |
| 201 | { |
| 202 | unsigned i; |
| 203 | int err; |
| 204 | |
| 205 | /* lookup all devices */ |
| 206 | for (i = 0; i < objio_seg->num_comps; i++) { |
| 207 | struct objio_dev_ent *ode; |
| 208 | |
| 209 | ode = _device_lookup(pnfslay, objio_seg, i, gfp_flags); |
| 210 | if (unlikely(IS_ERR(ode))) { |
| 211 | err = PTR_ERR(ode); |
| 212 | goto out; |
| 213 | } |
| 214 | objio_seg->ods[i] = ode; |
| 215 | } |
| 216 | err = 0; |
| 217 | |
| 218 | out: |
| 219 | dprintk("%s: return=%d\n", __func__, err); |
| 220 | return err; |
| 221 | } |
| 222 | |
Boaz Harrosh | 09f5bf4 | 2011-05-22 19:50:20 +0300 | [diff] [blame] | 223 | static int _verify_data_map(struct pnfs_osd_layout *layout) |
| 224 | { |
| 225 | struct pnfs_osd_data_map *data_map = &layout->olo_map; |
| 226 | u64 stripe_length; |
| 227 | u32 group_width; |
| 228 | |
| 229 | /* FIXME: Only raid0 for now. if not go through MDS */ |
| 230 | if (data_map->odm_raid_algorithm != PNFS_OSD_RAID_0) { |
| 231 | printk(KERN_ERR "Only RAID_0 for now\n"); |
| 232 | return -ENOTSUPP; |
| 233 | } |
| 234 | if (0 != (data_map->odm_num_comps % (data_map->odm_mirror_cnt + 1))) { |
| 235 | printk(KERN_ERR "Data Map wrong, num_comps=%u mirrors=%u\n", |
| 236 | data_map->odm_num_comps, data_map->odm_mirror_cnt); |
| 237 | return -EINVAL; |
| 238 | } |
| 239 | |
| 240 | if (data_map->odm_group_width) |
| 241 | group_width = data_map->odm_group_width; |
| 242 | else |
| 243 | group_width = data_map->odm_num_comps / |
| 244 | (data_map->odm_mirror_cnt + 1); |
| 245 | |
| 246 | stripe_length = (u64)data_map->odm_stripe_unit * group_width; |
| 247 | if (stripe_length >= (1ULL << 32)) { |
| 248 | printk(KERN_ERR "Total Stripe length(0x%llx)" |
| 249 | " >= 32bit is not supported\n", _LLU(stripe_length)); |
| 250 | return -ENOTSUPP; |
| 251 | } |
| 252 | |
| 253 | if (0 != (data_map->odm_stripe_unit & ~PAGE_MASK)) { |
| 254 | printk(KERN_ERR "Stripe Unit(0x%llx)" |
| 255 | " must be Multples of PAGE_SIZE(0x%lx)\n", |
| 256 | _LLU(data_map->odm_stripe_unit), PAGE_SIZE); |
| 257 | return -ENOTSUPP; |
| 258 | } |
| 259 | |
| 260 | return 0; |
| 261 | } |
| 262 | |
| 263 | static void copy_single_comp(struct pnfs_osd_object_cred *cur_comp, |
| 264 | struct pnfs_osd_object_cred *src_comp, |
| 265 | struct caps_buffers *caps_p) |
| 266 | { |
| 267 | WARN_ON(src_comp->oc_cap_key.cred_len > sizeof(caps_p->caps_key)); |
| 268 | WARN_ON(src_comp->oc_cap.cred_len > sizeof(caps_p->creds)); |
| 269 | |
| 270 | *cur_comp = *src_comp; |
| 271 | |
| 272 | memcpy(caps_p->caps_key, src_comp->oc_cap_key.cred, |
| 273 | sizeof(caps_p->caps_key)); |
| 274 | cur_comp->oc_cap_key.cred = caps_p->caps_key; |
| 275 | |
| 276 | memcpy(caps_p->creds, src_comp->oc_cap.cred, |
| 277 | sizeof(caps_p->creds)); |
| 278 | cur_comp->oc_cap.cred = caps_p->creds; |
| 279 | } |
| 280 | |
| 281 | int objio_alloc_lseg(struct pnfs_layout_segment **outp, |
| 282 | struct pnfs_layout_hdr *pnfslay, |
| 283 | struct pnfs_layout_range *range, |
| 284 | struct xdr_stream *xdr, |
| 285 | gfp_t gfp_flags) |
| 286 | { |
| 287 | struct objio_segment *objio_seg; |
| 288 | struct pnfs_osd_xdr_decode_layout_iter iter; |
| 289 | struct pnfs_osd_layout layout; |
| 290 | struct pnfs_osd_object_cred *cur_comp, src_comp; |
| 291 | struct caps_buffers *caps_p; |
| 292 | int err; |
| 293 | |
| 294 | err = pnfs_osd_xdr_decode_layout_map(&layout, &iter, xdr); |
| 295 | if (unlikely(err)) |
| 296 | return err; |
| 297 | |
| 298 | err = _verify_data_map(&layout); |
| 299 | if (unlikely(err)) |
| 300 | return err; |
| 301 | |
| 302 | objio_seg = kzalloc(sizeof(*objio_seg) + |
| 303 | sizeof(objio_seg->ods[0]) * layout.olo_num_comps + |
| 304 | sizeof(*objio_seg->comps) * layout.olo_num_comps + |
| 305 | sizeof(struct caps_buffers) * layout.olo_num_comps, |
| 306 | gfp_flags); |
| 307 | if (!objio_seg) |
| 308 | return -ENOMEM; |
| 309 | |
| 310 | objio_seg->comps = (void *)(objio_seg->ods + layout.olo_num_comps); |
| 311 | cur_comp = objio_seg->comps; |
| 312 | caps_p = (void *)(cur_comp + layout.olo_num_comps); |
| 313 | while (pnfs_osd_xdr_decode_layout_comp(&src_comp, &iter, xdr, &err)) |
| 314 | copy_single_comp(cur_comp++, &src_comp, caps_p++); |
| 315 | if (unlikely(err)) |
| 316 | goto err; |
| 317 | |
| 318 | objio_seg->num_comps = layout.olo_num_comps; |
| 319 | objio_seg->comps_index = layout.olo_comps_index; |
Boaz Harrosh | b6c05f1 | 2011-05-26 21:45:34 +0300 | [diff] [blame] | 320 | err = objio_devices_lookup(pnfslay, objio_seg, gfp_flags); |
| 321 | if (err) |
| 322 | goto err; |
Boaz Harrosh | 09f5bf4 | 2011-05-22 19:50:20 +0300 | [diff] [blame] | 323 | |
| 324 | objio_seg->mirrors_p1 = layout.olo_map.odm_mirror_cnt + 1; |
| 325 | objio_seg->stripe_unit = layout.olo_map.odm_stripe_unit; |
| 326 | if (layout.olo_map.odm_group_width) { |
| 327 | objio_seg->group_width = layout.olo_map.odm_group_width; |
| 328 | objio_seg->group_depth = layout.olo_map.odm_group_depth; |
| 329 | objio_seg->group_count = layout.olo_map.odm_num_comps / |
| 330 | objio_seg->mirrors_p1 / |
| 331 | objio_seg->group_width; |
| 332 | } else { |
| 333 | objio_seg->group_width = layout.olo_map.odm_num_comps / |
| 334 | objio_seg->mirrors_p1; |
| 335 | objio_seg->group_depth = -1; |
| 336 | objio_seg->group_count = 1; |
| 337 | } |
| 338 | |
| 339 | *outp = &objio_seg->lseg; |
| 340 | return 0; |
| 341 | |
| 342 | err: |
| 343 | kfree(objio_seg); |
| 344 | dprintk("%s: Error: return %d\n", __func__, err); |
| 345 | *outp = NULL; |
| 346 | return err; |
| 347 | } |
| 348 | |
| 349 | void objio_free_lseg(struct pnfs_layout_segment *lseg) |
| 350 | { |
Boaz Harrosh | b6c05f1 | 2011-05-26 21:45:34 +0300 | [diff] [blame] | 351 | int i; |
Boaz Harrosh | 09f5bf4 | 2011-05-22 19:50:20 +0300 | [diff] [blame] | 352 | struct objio_segment *objio_seg = OBJIO_LSEG(lseg); |
| 353 | |
Boaz Harrosh | b6c05f1 | 2011-05-26 21:45:34 +0300 | [diff] [blame] | 354 | for (i = 0; i < objio_seg->num_comps; i++) { |
| 355 | if (!objio_seg->ods[i]) |
| 356 | break; |
| 357 | nfs4_put_deviceid_node(&objio_seg->ods[i]->id_node); |
| 358 | } |
Boaz Harrosh | 09f5bf4 | 2011-05-22 19:50:20 +0300 | [diff] [blame] | 359 | kfree(objio_seg); |
| 360 | } |
| 361 | |
Benny Halevy | c93407d | 2011-05-22 19:49:06 +0300 | [diff] [blame] | 362 | |
| 363 | static struct pnfs_layoutdriver_type objlayout_type = { |
| 364 | .id = LAYOUT_OSD2_OBJECTS, |
| 365 | .name = "LAYOUT_OSD2_OBJECTS", |
Boaz Harrosh | 09f5bf4 | 2011-05-22 19:50:20 +0300 | [diff] [blame] | 366 | |
Benny Halevy | e51b841 | 2011-05-22 19:51:48 +0300 | [diff] [blame] | 367 | .alloc_layout_hdr = objlayout_alloc_layout_hdr, |
| 368 | .free_layout_hdr = objlayout_free_layout_hdr, |
| 369 | |
Boaz Harrosh | 09f5bf4 | 2011-05-22 19:50:20 +0300 | [diff] [blame] | 370 | .alloc_lseg = objlayout_alloc_lseg, |
| 371 | .free_lseg = objlayout_free_lseg, |
Boaz Harrosh | b6c05f1 | 2011-05-26 21:45:34 +0300 | [diff] [blame] | 372 | |
| 373 | .free_deviceid_node = objio_free_deviceid_node, |
Benny Halevy | c93407d | 2011-05-22 19:49:06 +0300 | [diff] [blame] | 374 | }; |
| 375 | |
| 376 | MODULE_DESCRIPTION("pNFS Layout Driver for OSD2 objects"); |
| 377 | MODULE_AUTHOR("Benny Halevy <bhalevy@panasas.com>"); |
| 378 | MODULE_LICENSE("GPL"); |
| 379 | |
| 380 | static int __init |
| 381 | objlayout_init(void) |
| 382 | { |
| 383 | int ret = pnfs_register_layoutdriver(&objlayout_type); |
| 384 | |
| 385 | if (ret) |
| 386 | printk(KERN_INFO |
| 387 | "%s: Registering OSD pNFS Layout Driver failed: error=%d\n", |
| 388 | __func__, ret); |
| 389 | else |
| 390 | printk(KERN_INFO "%s: Registered OSD pNFS Layout Driver\n", |
| 391 | __func__); |
| 392 | return ret; |
| 393 | } |
| 394 | |
| 395 | static void __exit |
| 396 | objlayout_exit(void) |
| 397 | { |
| 398 | pnfs_unregister_layoutdriver(&objlayout_type); |
| 399 | printk(KERN_INFO "%s: Unregistered OSD pNFS Layout Driver\n", |
| 400 | __func__); |
| 401 | } |
| 402 | |
| 403 | module_init(objlayout_init); |
| 404 | module_exit(objlayout_exit); |