blob: ee0ebff103a412bc3db69c05f0a30ecc1872be48 [file] [log] [blame]
Yu Zhaod1b054d2009-03-20 11:25:11 +08001/*
2 * drivers/pci/iov.c
3 *
4 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
5 *
6 * PCI Express I/O Virtualization (IOV) support.
7 * Single Root IOV 1.0
Yu Zhao302b4212009-05-18 13:51:32 +08008 * Address Translation Service 1.0
Yu Zhaod1b054d2009-03-20 11:25:11 +08009 */
10
11#include <linux/pci.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090012#include <linux/slab.h>
Yu Zhaod1b054d2009-03-20 11:25:11 +080013#include <linux/mutex.h>
Paul Gortmaker363c75d2011-05-27 09:37:25 -040014#include <linux/export.h>
Yu Zhaod1b054d2009-03-20 11:25:11 +080015#include <linux/string.h>
16#include <linux/delay.h>
Joerg Roedel5cdede22011-04-04 15:55:18 +020017#include <linux/pci-ats.h>
Yu Zhaod1b054d2009-03-20 11:25:11 +080018#include "pci.h"
19
Yu Zhaodd7cc442009-03-20 11:25:15 +080020#define VIRTFN_ID_LEN 16
Yu Zhaod1b054d2009-03-20 11:25:11 +080021
Wei Yangb07579c2015-03-25 16:23:48 +080022int pci_iov_virtfn_bus(struct pci_dev *dev, int vf_id)
Yu Zhaoa28724b2009-03-20 11:25:13 +080023{
Wei Yangb07579c2015-03-25 16:23:48 +080024 if (!dev->is_physfn)
25 return -EINVAL;
Yu Zhaoa28724b2009-03-20 11:25:13 +080026 return dev->bus->number + ((dev->devfn + dev->sriov->offset +
Wei Yangb07579c2015-03-25 16:23:48 +080027 dev->sriov->stride * vf_id) >> 8);
Yu Zhaoa28724b2009-03-20 11:25:13 +080028}
29
Wei Yangb07579c2015-03-25 16:23:48 +080030int pci_iov_virtfn_devfn(struct pci_dev *dev, int vf_id)
Yu Zhaoa28724b2009-03-20 11:25:13 +080031{
Wei Yangb07579c2015-03-25 16:23:48 +080032 if (!dev->is_physfn)
33 return -EINVAL;
Yu Zhaoa28724b2009-03-20 11:25:13 +080034 return (dev->devfn + dev->sriov->offset +
Wei Yangb07579c2015-03-25 16:23:48 +080035 dev->sriov->stride * vf_id) & 0xff;
Yu Zhaoa28724b2009-03-20 11:25:13 +080036}
37
Wei Yangf59dca22015-03-25 16:23:46 +080038/*
39 * Per SR-IOV spec sec 3.3.10 and 3.3.11, First VF Offset and VF Stride may
40 * change when NumVFs changes.
41 *
42 * Update iov->offset and iov->stride when NumVFs is written.
43 */
44static inline void pci_iov_set_numvfs(struct pci_dev *dev, int nr_virtfn)
45{
46 struct pci_sriov *iov = dev->sriov;
47
48 pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, nr_virtfn);
49 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &iov->offset);
50 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &iov->stride);
51}
52
Wei Yang4449f072015-03-25 16:23:47 +080053/*
54 * The PF consumes one bus number. NumVFs, First VF Offset, and VF Stride
55 * determine how many additional bus numbers will be consumed by VFs.
56 *
57 * Iterate over all valid NumVFs and calculate the maximum number of bus
58 * numbers that could ever be required.
59 */
60static inline u8 virtfn_max_buses(struct pci_dev *dev)
61{
62 struct pci_sriov *iov = dev->sriov;
63 int nr_virtfn;
64 u8 max = 0;
Wei Yangb07579c2015-03-25 16:23:48 +080065 int busnr;
Wei Yang4449f072015-03-25 16:23:47 +080066
67 for (nr_virtfn = 1; nr_virtfn <= iov->total_VFs; nr_virtfn++) {
68 pci_iov_set_numvfs(dev, nr_virtfn);
Wei Yangb07579c2015-03-25 16:23:48 +080069 busnr = pci_iov_virtfn_bus(dev, nr_virtfn - 1);
Wei Yang4449f072015-03-25 16:23:47 +080070 if (busnr > max)
71 max = busnr;
72 }
73
74 return max;
75}
76
Yu Zhaodd7cc442009-03-20 11:25:15 +080077static struct pci_bus *virtfn_add_bus(struct pci_bus *bus, int busnr)
78{
Yu Zhaodd7cc442009-03-20 11:25:15 +080079 struct pci_bus *child;
80
81 if (bus->number == busnr)
82 return bus;
83
84 child = pci_find_bus(pci_domain_nr(bus), busnr);
85 if (child)
86 return child;
87
88 child = pci_add_new_bus(bus, NULL, busnr);
89 if (!child)
90 return NULL;
91
Yinghai Lub7eac052012-05-17 18:51:13 -070092 pci_bus_insert_busn_res(child, busnr, busnr);
Yu Zhaodd7cc442009-03-20 11:25:15 +080093
94 return child;
95}
96
Jiang Liudc087f22013-05-25 21:48:37 +080097static void virtfn_remove_bus(struct pci_bus *physbus, struct pci_bus *virtbus)
Yu Zhaodd7cc442009-03-20 11:25:15 +080098{
Jiang Liudc087f22013-05-25 21:48:37 +080099 if (physbus != virtbus && list_empty(&virtbus->devices))
100 pci_remove_bus(virtbus);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800101}
102
Wei Yang0e6c9122015-03-25 16:23:44 +0800103resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno)
104{
105 if (!dev->is_physfn)
106 return 0;
107
108 return dev->sriov->barsz[resno - PCI_IOV_RESOURCES];
109}
110
Yu Zhaodd7cc442009-03-20 11:25:15 +0800111static int virtfn_add(struct pci_dev *dev, int id, int reset)
112{
113 int i;
Jiang Liudc087f22013-05-25 21:48:37 +0800114 int rc = -ENOMEM;
Yu Zhaodd7cc442009-03-20 11:25:15 +0800115 u64 size;
116 char buf[VIRTFN_ID_LEN];
117 struct pci_dev *virtfn;
118 struct resource *res;
119 struct pci_sriov *iov = dev->sriov;
Gu Zheng8b1fce02013-05-25 21:48:31 +0800120 struct pci_bus *bus;
Yu Zhaodd7cc442009-03-20 11:25:15 +0800121
122 mutex_lock(&iov->dev->sriov->lock);
Wei Yangb07579c2015-03-25 16:23:48 +0800123 bus = virtfn_add_bus(dev->bus, pci_iov_virtfn_bus(dev, id));
Jiang Liudc087f22013-05-25 21:48:37 +0800124 if (!bus)
125 goto failed;
126
127 virtfn = pci_alloc_dev(bus);
128 if (!virtfn)
129 goto failed0;
130
Wei Yangb07579c2015-03-25 16:23:48 +0800131 virtfn->devfn = pci_iov_virtfn_devfn(dev, id);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800132 virtfn->vendor = dev->vendor;
133 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_DID, &virtfn->device);
134 pci_setup_device(virtfn);
135 virtfn->dev.parent = dev->dev.parent;
Xudong Haofbf33f52013-05-31 12:21:29 +0800136 virtfn->physfn = pci_dev_get(dev);
137 virtfn->is_virtfn = 1;
Alex Williamsonaa9319772014-01-09 08:36:08 -0700138 virtfn->multifunction = 0;
Yu Zhaodd7cc442009-03-20 11:25:15 +0800139
140 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
Bjorn Helgaasc1fe1f92015-03-25 16:23:45 +0800141 res = &dev->resource[i + PCI_IOV_RESOURCES];
Yu Zhaodd7cc442009-03-20 11:25:15 +0800142 if (!res->parent)
143 continue;
144 virtfn->resource[i].name = pci_name(virtfn);
145 virtfn->resource[i].flags = res->flags;
Wei Yang0e6c9122015-03-25 16:23:44 +0800146 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800147 virtfn->resource[i].start = res->start + size * id;
148 virtfn->resource[i].end = virtfn->resource[i].start + size - 1;
149 rc = request_resource(res, &virtfn->resource[i]);
150 BUG_ON(rc);
151 }
152
153 if (reset)
Yu Zhao8c1c6992009-06-13 15:52:13 +0800154 __pci_reset_function(virtfn);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800155
156 pci_device_add(virtfn, virtfn->bus);
157 mutex_unlock(&iov->dev->sriov->lock);
158
Yijing Wangc893d132014-05-30 11:01:03 +0800159 pci_bus_add_device(virtfn);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800160 sprintf(buf, "virtfn%u", id);
161 rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf);
162 if (rc)
163 goto failed1;
164 rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn");
165 if (rc)
166 goto failed2;
167
168 kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE);
169
170 return 0;
171
172failed2:
173 sysfs_remove_link(&dev->dev.kobj, buf);
174failed1:
175 pci_dev_put(dev);
176 mutex_lock(&iov->dev->sriov->lock);
Yinghai Lu210647a2012-02-25 13:54:20 -0800177 pci_stop_and_remove_bus_device(virtfn);
Jiang Liudc087f22013-05-25 21:48:37 +0800178failed0:
179 virtfn_remove_bus(dev->bus, bus);
180failed:
Yu Zhaodd7cc442009-03-20 11:25:15 +0800181 mutex_unlock(&iov->dev->sriov->lock);
182
183 return rc;
184}
185
186static void virtfn_remove(struct pci_dev *dev, int id, int reset)
187{
188 char buf[VIRTFN_ID_LEN];
Yu Zhaodd7cc442009-03-20 11:25:15 +0800189 struct pci_dev *virtfn;
190 struct pci_sriov *iov = dev->sriov;
191
Jiang Liudc087f22013-05-25 21:48:37 +0800192 virtfn = pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus),
Wei Yangb07579c2015-03-25 16:23:48 +0800193 pci_iov_virtfn_bus(dev, id),
194 pci_iov_virtfn_devfn(dev, id));
Yu Zhaodd7cc442009-03-20 11:25:15 +0800195 if (!virtfn)
196 return;
197
Yu Zhaodd7cc442009-03-20 11:25:15 +0800198 if (reset) {
199 device_release_driver(&virtfn->dev);
Yu Zhao8c1c6992009-06-13 15:52:13 +0800200 __pci_reset_function(virtfn);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800201 }
202
203 sprintf(buf, "virtfn%u", id);
204 sysfs_remove_link(&dev->dev.kobj, buf);
Yinghai Lu09cedbe2012-02-04 22:55:01 -0800205 /*
206 * pci_stop_dev() could have been called for this virtfn already,
207 * so the directory for the virtfn may have been removed before.
208 * Double check to avoid spurious sysfs warnings.
209 */
210 if (virtfn->dev.kobj.sd)
211 sysfs_remove_link(&virtfn->dev.kobj, "physfn");
Yu Zhaodd7cc442009-03-20 11:25:15 +0800212
213 mutex_lock(&iov->dev->sriov->lock);
Yinghai Lu210647a2012-02-25 13:54:20 -0800214 pci_stop_and_remove_bus_device(virtfn);
Jiang Liudc087f22013-05-25 21:48:37 +0800215 virtfn_remove_bus(dev->bus, virtfn->bus);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800216 mutex_unlock(&iov->dev->sriov->lock);
217
Jiang Liudc087f22013-05-25 21:48:37 +0800218 /* balance pci_get_domain_bus_and_slot() */
219 pci_dev_put(virtfn);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800220 pci_dev_put(dev);
221}
222
Wei Yang995df522015-03-25 16:23:49 +0800223int __weak pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
224{
225 return 0;
226}
227
Yu Zhaodd7cc442009-03-20 11:25:15 +0800228static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
229{
230 int rc;
231 int i, j;
232 int nres;
233 u16 offset, stride, initial;
234 struct resource *res;
235 struct pci_dev *pdev;
236 struct pci_sriov *iov = dev->sriov;
Ram Paibbef98a2011-11-06 10:33:10 +0800237 int bars = 0;
Wei Yangb07579c2015-03-25 16:23:48 +0800238 int bus;
Wei Yang995df522015-03-25 16:23:49 +0800239 int retval;
Yu Zhaodd7cc442009-03-20 11:25:15 +0800240
241 if (!nr_virtfn)
242 return 0;
243
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700244 if (iov->num_VFs)
Yu Zhaodd7cc442009-03-20 11:25:15 +0800245 return -EINVAL;
246
247 pci_read_config_word(dev, iov->pos + PCI_SRIOV_INITIAL_VF, &initial);
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700248 if (initial > iov->total_VFs ||
249 (!(iov->cap & PCI_SRIOV_CAP_VFM) && (initial != iov->total_VFs)))
Yu Zhaodd7cc442009-03-20 11:25:15 +0800250 return -EIO;
251
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700252 if (nr_virtfn < 0 || nr_virtfn > iov->total_VFs ||
Yu Zhaodd7cc442009-03-20 11:25:15 +0800253 (!(iov->cap & PCI_SRIOV_CAP_VFM) && (nr_virtfn > initial)))
254 return -EINVAL;
255
Yu Zhaodd7cc442009-03-20 11:25:15 +0800256 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &offset);
257 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &stride);
258 if (!offset || (nr_virtfn > 1 && !stride))
259 return -EIO;
260
261 nres = 0;
262 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
Ram Paibbef98a2011-11-06 10:33:10 +0800263 bars |= (1 << (i + PCI_IOV_RESOURCES));
Bjorn Helgaasc1fe1f92015-03-25 16:23:45 +0800264 res = &dev->resource[i + PCI_IOV_RESOURCES];
Yu Zhaodd7cc442009-03-20 11:25:15 +0800265 if (res->parent)
266 nres++;
267 }
268 if (nres != iov->nres) {
269 dev_err(&dev->dev, "not enough MMIO resources for SR-IOV\n");
270 return -ENOMEM;
271 }
272
273 iov->offset = offset;
274 iov->stride = stride;
275
Wei Yangb07579c2015-03-25 16:23:48 +0800276 bus = pci_iov_virtfn_bus(dev, nr_virtfn - 1);
Bjorn Helgaas68f8e9f2015-03-25 16:23:42 +0800277 if (bus > dev->bus->busn_res.end) {
278 dev_err(&dev->dev, "can't enable %d VFs (bus %02x out of range of %pR)\n",
279 nr_virtfn, bus, &dev->bus->busn_res);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800280 return -ENOMEM;
281 }
282
Ram Paibbef98a2011-11-06 10:33:10 +0800283 if (pci_enable_resources(dev, bars)) {
284 dev_err(&dev->dev, "SR-IOV: IOV BARS not allocated\n");
285 return -ENOMEM;
286 }
287
Yu Zhaodd7cc442009-03-20 11:25:15 +0800288 if (iov->link != dev->devfn) {
289 pdev = pci_get_slot(dev->bus, iov->link);
290 if (!pdev)
291 return -ENODEV;
292
Jiang Liudc087f22013-05-25 21:48:37 +0800293 if (!pdev->is_physfn) {
294 pci_dev_put(pdev);
Stefan Assmann652d1102013-07-31 16:47:56 -0600295 return -ENOSYS;
Jiang Liudc087f22013-05-25 21:48:37 +0800296 }
Yu Zhaodd7cc442009-03-20 11:25:15 +0800297
298 rc = sysfs_create_link(&dev->dev.kobj,
299 &pdev->dev.kobj, "dep_link");
Jiang Liudc087f22013-05-25 21:48:37 +0800300 pci_dev_put(pdev);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800301 if (rc)
302 return rc;
303 }
304
Wei Yangf59dca22015-03-25 16:23:46 +0800305 pci_iov_set_numvfs(dev, nr_virtfn);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800306 iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE;
Jan Kiszkafb51ccb2011-11-04 09:45:59 +0100307 pci_cfg_access_lock(dev);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800308 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
309 msleep(100);
Jan Kiszkafb51ccb2011-11-04 09:45:59 +0100310 pci_cfg_access_unlock(dev);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800311
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700312 iov->initial_VFs = initial;
Yu Zhaodd7cc442009-03-20 11:25:15 +0800313 if (nr_virtfn < initial)
314 initial = nr_virtfn;
315
Wei Yang995df522015-03-25 16:23:49 +0800316 if ((retval = pcibios_sriov_enable(dev, initial))) {
317 dev_err(&dev->dev, "failure %d from pcibios_sriov_enable()\n",
318 retval);
319 return retval;
320 }
321
Yu Zhaodd7cc442009-03-20 11:25:15 +0800322 for (i = 0; i < initial; i++) {
323 rc = virtfn_add(dev, i, 0);
324 if (rc)
325 goto failed;
326 }
327
328 kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE);
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700329 iov->num_VFs = nr_virtfn;
Yu Zhaodd7cc442009-03-20 11:25:15 +0800330
331 return 0;
332
333failed:
334 for (j = 0; j < i; j++)
335 virtfn_remove(dev, j, 0);
336
337 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
Jan Kiszkafb51ccb2011-11-04 09:45:59 +0100338 pci_cfg_access_lock(dev);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800339 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
Wei Yangf59dca22015-03-25 16:23:46 +0800340 pci_iov_set_numvfs(dev, 0);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800341 ssleep(1);
Jan Kiszkafb51ccb2011-11-04 09:45:59 +0100342 pci_cfg_access_unlock(dev);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800343
344 if (iov->link != dev->devfn)
345 sysfs_remove_link(&dev->dev.kobj, "dep_link");
346
347 return rc;
348}
349
Wei Yang995df522015-03-25 16:23:49 +0800350int __weak pcibios_sriov_disable(struct pci_dev *pdev)
351{
352 return 0;
353}
354
Yu Zhaodd7cc442009-03-20 11:25:15 +0800355static void sriov_disable(struct pci_dev *dev)
356{
357 int i;
358 struct pci_sriov *iov = dev->sriov;
359
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700360 if (!iov->num_VFs)
Yu Zhaodd7cc442009-03-20 11:25:15 +0800361 return;
362
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700363 for (i = 0; i < iov->num_VFs; i++)
Yu Zhaodd7cc442009-03-20 11:25:15 +0800364 virtfn_remove(dev, i, 0);
365
Wei Yang995df522015-03-25 16:23:49 +0800366 pcibios_sriov_disable(dev);
367
Yu Zhaodd7cc442009-03-20 11:25:15 +0800368 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
Jan Kiszkafb51ccb2011-11-04 09:45:59 +0100369 pci_cfg_access_lock(dev);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800370 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
371 ssleep(1);
Jan Kiszkafb51ccb2011-11-04 09:45:59 +0100372 pci_cfg_access_unlock(dev);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800373
374 if (iov->link != dev->devfn)
375 sysfs_remove_link(&dev->dev.kobj, "dep_link");
376
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700377 iov->num_VFs = 0;
Wei Yangf59dca22015-03-25 16:23:46 +0800378 pci_iov_set_numvfs(dev, 0);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800379}
380
Yu Zhaod1b054d2009-03-20 11:25:11 +0800381static int sriov_init(struct pci_dev *dev, int pos)
382{
Wei Yang0e6c9122015-03-25 16:23:44 +0800383 int i, bar64;
Yu Zhaod1b054d2009-03-20 11:25:11 +0800384 int rc;
385 int nres;
386 u32 pgsz;
387 u16 ctrl, total, offset, stride;
388 struct pci_sriov *iov;
389 struct resource *res;
390 struct pci_dev *pdev;
391
Yijing Wang62f87c02012-07-24 17:20:03 +0800392 if (pci_pcie_type(dev) != PCI_EXP_TYPE_RC_END &&
393 pci_pcie_type(dev) != PCI_EXP_TYPE_ENDPOINT)
Yu Zhaod1b054d2009-03-20 11:25:11 +0800394 return -ENODEV;
395
396 pci_read_config_word(dev, pos + PCI_SRIOV_CTRL, &ctrl);
397 if (ctrl & PCI_SRIOV_CTRL_VFE) {
398 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, 0);
399 ssleep(1);
400 }
401
402 pci_read_config_word(dev, pos + PCI_SRIOV_TOTAL_VF, &total);
403 if (!total)
404 return 0;
405
406 ctrl = 0;
407 list_for_each_entry(pdev, &dev->bus->devices, bus_list)
408 if (pdev->is_physfn)
409 goto found;
410
411 pdev = NULL;
412 if (pci_ari_enabled(dev->bus))
413 ctrl |= PCI_SRIOV_CTRL_ARI;
414
415found:
416 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, ctrl);
ethan.zhao045cc222013-11-06 22:49:13 +0800417 pci_write_config_word(dev, pos + PCI_SRIOV_NUM_VF, 0);
Yu Zhaod1b054d2009-03-20 11:25:11 +0800418 pci_read_config_word(dev, pos + PCI_SRIOV_VF_OFFSET, &offset);
419 pci_read_config_word(dev, pos + PCI_SRIOV_VF_STRIDE, &stride);
420 if (!offset || (total > 1 && !stride))
421 return -EIO;
422
423 pci_read_config_dword(dev, pos + PCI_SRIOV_SUP_PGSIZE, &pgsz);
424 i = PAGE_SHIFT > 12 ? PAGE_SHIFT - 12 : 0;
425 pgsz &= ~((1 << i) - 1);
426 if (!pgsz)
427 return -EIO;
428
429 pgsz &= ~(pgsz - 1);
Vaidyanathan Srinivasan8161fe92012-02-02 23:11:20 +0530430 pci_write_config_dword(dev, pos + PCI_SRIOV_SYS_PGSIZE, pgsz);
Yu Zhaod1b054d2009-03-20 11:25:11 +0800431
Wei Yang0e6c9122015-03-25 16:23:44 +0800432 iov = kzalloc(sizeof(*iov), GFP_KERNEL);
433 if (!iov)
434 return -ENOMEM;
435
Yu Zhaod1b054d2009-03-20 11:25:11 +0800436 nres = 0;
437 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
Bjorn Helgaasc1fe1f92015-03-25 16:23:45 +0800438 res = &dev->resource[i + PCI_IOV_RESOURCES];
Wei Yang0e6c9122015-03-25 16:23:44 +0800439 bar64 = __pci_read_base(dev, pci_bar_unknown, res,
440 pos + PCI_SRIOV_BAR + i * 4);
Yu Zhaod1b054d2009-03-20 11:25:11 +0800441 if (!res->flags)
442 continue;
443 if (resource_size(res) & (PAGE_SIZE - 1)) {
444 rc = -EIO;
445 goto failed;
446 }
Wei Yang0e6c9122015-03-25 16:23:44 +0800447 iov->barsz[i] = resource_size(res);
Yu Zhaod1b054d2009-03-20 11:25:11 +0800448 res->end = res->start + resource_size(res) * total - 1;
Wei Yange88ae012015-03-25 16:23:43 +0800449 dev_info(&dev->dev, "VF(n) BAR%d space: %pR (contains BAR%d for %d VFs)\n",
450 i, res, i, total);
Wei Yang0e6c9122015-03-25 16:23:44 +0800451 i += bar64;
Yu Zhaod1b054d2009-03-20 11:25:11 +0800452 nres++;
453 }
454
Yu Zhaod1b054d2009-03-20 11:25:11 +0800455 iov->pos = pos;
456 iov->nres = nres;
457 iov->ctrl = ctrl;
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700458 iov->total_VFs = total;
Yu Zhaod1b054d2009-03-20 11:25:11 +0800459 iov->offset = offset;
460 iov->stride = stride;
461 iov->pgsz = pgsz;
462 iov->self = dev;
463 pci_read_config_dword(dev, pos + PCI_SRIOV_CAP, &iov->cap);
464 pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link);
Yijing Wang62f87c02012-07-24 17:20:03 +0800465 if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END)
Yu Zhao4d135db2009-05-20 17:11:57 +0800466 iov->link = PCI_DEVFN(PCI_SLOT(dev->devfn), iov->link);
Yu Zhaod1b054d2009-03-20 11:25:11 +0800467
468 if (pdev)
469 iov->dev = pci_dev_get(pdev);
Yu Zhaoe277d2f2009-05-18 13:51:33 +0800470 else
Yu Zhaod1b054d2009-03-20 11:25:11 +0800471 iov->dev = dev;
Yu Zhaoe277d2f2009-05-18 13:51:33 +0800472
473 mutex_init(&iov->lock);
Yu Zhaod1b054d2009-03-20 11:25:11 +0800474
475 dev->sriov = iov;
476 dev->is_physfn = 1;
Wei Yang4449f072015-03-25 16:23:47 +0800477 iov->max_VF_buses = virtfn_max_buses(dev);
Yu Zhaod1b054d2009-03-20 11:25:11 +0800478
479 return 0;
480
481failed:
482 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
Bjorn Helgaasc1fe1f92015-03-25 16:23:45 +0800483 res = &dev->resource[i + PCI_IOV_RESOURCES];
Yu Zhaod1b054d2009-03-20 11:25:11 +0800484 res->flags = 0;
485 }
486
Wei Yang0e6c9122015-03-25 16:23:44 +0800487 kfree(iov);
Yu Zhaod1b054d2009-03-20 11:25:11 +0800488 return rc;
489}
490
491static void sriov_release(struct pci_dev *dev)
492{
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700493 BUG_ON(dev->sriov->num_VFs);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800494
Yu Zhaoe277d2f2009-05-18 13:51:33 +0800495 if (dev != dev->sriov->dev)
Yu Zhaod1b054d2009-03-20 11:25:11 +0800496 pci_dev_put(dev->sriov->dev);
497
Yu Zhaoe277d2f2009-05-18 13:51:33 +0800498 mutex_destroy(&dev->sriov->lock);
499
Yu Zhaod1b054d2009-03-20 11:25:11 +0800500 kfree(dev->sriov);
501 dev->sriov = NULL;
502}
503
Yu Zhao8c5cdb62009-03-20 11:25:12 +0800504static void sriov_restore_state(struct pci_dev *dev)
505{
506 int i;
507 u16 ctrl;
508 struct pci_sriov *iov = dev->sriov;
509
510 pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &ctrl);
511 if (ctrl & PCI_SRIOV_CTRL_VFE)
512 return;
513
514 for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++)
515 pci_update_resource(dev, i);
516
517 pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz);
Wei Yangf59dca22015-03-25 16:23:46 +0800518 pci_iov_set_numvfs(dev, iov->num_VFs);
Yu Zhao8c5cdb62009-03-20 11:25:12 +0800519 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
520 if (iov->ctrl & PCI_SRIOV_CTRL_VFE)
521 msleep(100);
522}
523
Yu Zhaod1b054d2009-03-20 11:25:11 +0800524/**
525 * pci_iov_init - initialize the IOV capability
526 * @dev: the PCI device
527 *
528 * Returns 0 on success, or negative on failure.
529 */
530int pci_iov_init(struct pci_dev *dev)
531{
532 int pos;
533
Kenji Kaneshige5f4d91a2009-11-11 14:36:17 +0900534 if (!pci_is_pcie(dev))
Yu Zhaod1b054d2009-03-20 11:25:11 +0800535 return -ENODEV;
536
537 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV);
538 if (pos)
539 return sriov_init(dev, pos);
540
541 return -ENODEV;
542}
543
544/**
545 * pci_iov_release - release resources used by the IOV capability
546 * @dev: the PCI device
547 */
548void pci_iov_release(struct pci_dev *dev)
549{
550 if (dev->is_physfn)
551 sriov_release(dev);
552}
553
554/**
555 * pci_iov_resource_bar - get position of the SR-IOV BAR
556 * @dev: the PCI device
557 * @resno: the resource number
Yu Zhaod1b054d2009-03-20 11:25:11 +0800558 *
559 * Returns position of the BAR encapsulated in the SR-IOV capability.
560 */
Myron Stowe26ff46c2014-11-11 08:04:50 -0700561int pci_iov_resource_bar(struct pci_dev *dev, int resno)
Yu Zhaod1b054d2009-03-20 11:25:11 +0800562{
563 if (resno < PCI_IOV_RESOURCES || resno > PCI_IOV_RESOURCE_END)
564 return 0;
565
566 BUG_ON(!dev->is_physfn);
567
Yu Zhaod1b054d2009-03-20 11:25:11 +0800568 return dev->sriov->pos + PCI_SRIOV_BAR +
569 4 * (resno - PCI_IOV_RESOURCES);
570}
Yu Zhao8c5cdb62009-03-20 11:25:12 +0800571
Wei Yang978d2d62015-03-25 16:23:50 +0800572resource_size_t __weak pcibios_iov_resource_alignment(struct pci_dev *dev,
573 int resno)
574{
575 return pci_iov_resource_size(dev, resno);
576}
577
Yu Zhao8c5cdb62009-03-20 11:25:12 +0800578/**
Chris Wright6faf17f2009-08-28 13:00:06 -0700579 * pci_sriov_resource_alignment - get resource alignment for VF BAR
580 * @dev: the PCI device
581 * @resno: the resource number
582 *
583 * Returns the alignment of the VF BAR found in the SR-IOV capability.
584 * This is not the same as the resource size which is defined as
585 * the VF BAR size multiplied by the number of VFs. The alignment
586 * is just the VF BAR size.
587 */
Cam Macdonell0e522472010-09-07 17:25:20 -0700588resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno)
Chris Wright6faf17f2009-08-28 13:00:06 -0700589{
Wei Yang978d2d62015-03-25 16:23:50 +0800590 return pcibios_iov_resource_alignment(dev, resno);
Chris Wright6faf17f2009-08-28 13:00:06 -0700591}
592
593/**
Yu Zhao8c5cdb62009-03-20 11:25:12 +0800594 * pci_restore_iov_state - restore the state of the IOV capability
595 * @dev: the PCI device
596 */
597void pci_restore_iov_state(struct pci_dev *dev)
598{
599 if (dev->is_physfn)
600 sriov_restore_state(dev);
601}
Yu Zhaoa28724b2009-03-20 11:25:13 +0800602
603/**
604 * pci_iov_bus_range - find bus range used by Virtual Function
605 * @bus: the PCI bus
606 *
607 * Returns max number of buses (exclude current one) used by Virtual
608 * Functions.
609 */
610int pci_iov_bus_range(struct pci_bus *bus)
611{
612 int max = 0;
Yu Zhaoa28724b2009-03-20 11:25:13 +0800613 struct pci_dev *dev;
614
615 list_for_each_entry(dev, &bus->devices, bus_list) {
616 if (!dev->is_physfn)
617 continue;
Wei Yang4449f072015-03-25 16:23:47 +0800618 if (dev->sriov->max_VF_buses > max)
619 max = dev->sriov->max_VF_buses;
Yu Zhaoa28724b2009-03-20 11:25:13 +0800620 }
621
622 return max ? max - bus->number : 0;
623}
Yu Zhaodd7cc442009-03-20 11:25:15 +0800624
625/**
626 * pci_enable_sriov - enable the SR-IOV capability
627 * @dev: the PCI device
Randy Dunlap52a88732009-04-01 17:45:30 -0700628 * @nr_virtfn: number of virtual functions to enable
Yu Zhaodd7cc442009-03-20 11:25:15 +0800629 *
630 * Returns 0 on success, or negative on failure.
631 */
632int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn)
633{
634 might_sleep();
635
636 if (!dev->is_physfn)
Stefan Assmann652d1102013-07-31 16:47:56 -0600637 return -ENOSYS;
Yu Zhaodd7cc442009-03-20 11:25:15 +0800638
639 return sriov_enable(dev, nr_virtfn);
640}
641EXPORT_SYMBOL_GPL(pci_enable_sriov);
642
643/**
644 * pci_disable_sriov - disable the SR-IOV capability
645 * @dev: the PCI device
646 */
647void pci_disable_sriov(struct pci_dev *dev)
648{
649 might_sleep();
650
651 if (!dev->is_physfn)
652 return;
653
654 sriov_disable(dev);
655}
656EXPORT_SYMBOL_GPL(pci_disable_sriov);
Yu Zhao74bb1bc2009-03-20 11:25:16 +0800657
658/**
Williams, Mitch Afb8a0d92010-02-10 01:43:04 +0000659 * pci_num_vf - return number of VFs associated with a PF device_release_driver
660 * @dev: the PCI device
661 *
662 * Returns number of VFs, or 0 if SR-IOV is not enabled.
663 */
664int pci_num_vf(struct pci_dev *dev)
665{
Bjorn Helgaas1452cd72012-11-09 20:35:01 -0700666 if (!dev->is_physfn)
Williams, Mitch Afb8a0d92010-02-10 01:43:04 +0000667 return 0;
Bjorn Helgaas1452cd72012-11-09 20:35:01 -0700668
669 return dev->sriov->num_VFs;
Williams, Mitch Afb8a0d92010-02-10 01:43:04 +0000670}
671EXPORT_SYMBOL_GPL(pci_num_vf);
Donald Dutilebff73152012-11-05 15:20:37 -0500672
673/**
Alexander Duyck5a8eb242013-04-25 04:42:29 +0000674 * pci_vfs_assigned - returns number of VFs are assigned to a guest
675 * @dev: the PCI device
676 *
677 * Returns number of VFs belonging to this device that are assigned to a guest.
Stefan Assmann652d1102013-07-31 16:47:56 -0600678 * If device is not a physical function returns 0.
Alexander Duyck5a8eb242013-04-25 04:42:29 +0000679 */
680int pci_vfs_assigned(struct pci_dev *dev)
681{
682 struct pci_dev *vfdev;
683 unsigned int vfs_assigned = 0;
684 unsigned short dev_id;
685
686 /* only search if we are a PF */
687 if (!dev->is_physfn)
688 return 0;
689
690 /*
691 * determine the device ID for the VFs, the vendor ID will be the
692 * same as the PF so there is no need to check for that one
693 */
694 pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_VF_DID, &dev_id);
695
696 /* loop through all the VFs to see if we own any that are assigned */
697 vfdev = pci_get_device(dev->vendor, dev_id, NULL);
698 while (vfdev) {
699 /*
700 * It is considered assigned if it is a virtual function with
701 * our dev as the physical function and the assigned bit is set
702 */
703 if (vfdev->is_virtfn && (vfdev->physfn == dev) &&
Ethan Zhaobe634972014-09-09 10:21:28 +0800704 pci_is_dev_assigned(vfdev))
Alexander Duyck5a8eb242013-04-25 04:42:29 +0000705 vfs_assigned++;
706
707 vfdev = pci_get_device(dev->vendor, dev_id, vfdev);
708 }
709
710 return vfs_assigned;
711}
712EXPORT_SYMBOL_GPL(pci_vfs_assigned);
713
714/**
Donald Dutilebff73152012-11-05 15:20:37 -0500715 * pci_sriov_set_totalvfs -- reduce the TotalVFs available
716 * @dev: the PCI PF device
Randy Dunlap2094f162013-01-09 17:12:52 -0800717 * @numvfs: number that should be used for TotalVFs supported
Donald Dutilebff73152012-11-05 15:20:37 -0500718 *
719 * Should be called from PF driver's probe routine with
720 * device's mutex held.
721 *
722 * Returns 0 if PF is an SRIOV-capable device and
Stefan Assmann652d1102013-07-31 16:47:56 -0600723 * value of numvfs valid. If not a PF return -ENOSYS;
724 * if numvfs is invalid return -EINVAL;
Donald Dutilebff73152012-11-05 15:20:37 -0500725 * if VFs already enabled, return -EBUSY.
726 */
727int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs)
728{
Stefan Assmann652d1102013-07-31 16:47:56 -0600729 if (!dev->is_physfn)
730 return -ENOSYS;
731 if (numvfs > dev->sriov->total_VFs)
Donald Dutilebff73152012-11-05 15:20:37 -0500732 return -EINVAL;
733
734 /* Shouldn't change if VFs already enabled */
735 if (dev->sriov->ctrl & PCI_SRIOV_CTRL_VFE)
736 return -EBUSY;
737 else
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700738 dev->sriov->driver_max_VFs = numvfs;
Donald Dutilebff73152012-11-05 15:20:37 -0500739
740 return 0;
741}
742EXPORT_SYMBOL_GPL(pci_sriov_set_totalvfs);
743
744/**
Jonghwan Choiddc191f2013-07-08 14:02:43 -0600745 * pci_sriov_get_totalvfs -- get total VFs supported on this device
Donald Dutilebff73152012-11-05 15:20:37 -0500746 * @dev: the PCI PF device
747 *
748 * For a PCIe device with SRIOV support, return the PCIe
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700749 * SRIOV capability value of TotalVFs or the value of driver_max_VFs
Stefan Assmann652d1102013-07-31 16:47:56 -0600750 * if the driver reduced it. Otherwise 0.
Donald Dutilebff73152012-11-05 15:20:37 -0500751 */
752int pci_sriov_get_totalvfs(struct pci_dev *dev)
753{
Bjorn Helgaas1452cd72012-11-09 20:35:01 -0700754 if (!dev->is_physfn)
Stefan Assmann652d1102013-07-31 16:47:56 -0600755 return 0;
Donald Dutilebff73152012-11-05 15:20:37 -0500756
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700757 if (dev->sriov->driver_max_VFs)
758 return dev->sriov->driver_max_VFs;
Bjorn Helgaas1452cd72012-11-09 20:35:01 -0700759
760 return dev->sriov->total_VFs;
Donald Dutilebff73152012-11-05 15:20:37 -0500761}
762EXPORT_SYMBOL_GPL(pci_sriov_get_totalvfs);