blob: 47227820406d17f3e33f921effac2210e38ffd61 [file] [log] [blame]
Yu Zhaod1b054d2009-03-20 11:25:11 +08001/*
2 * drivers/pci/iov.c
3 *
4 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
5 *
6 * PCI Express I/O Virtualization (IOV) support.
7 * Single Root IOV 1.0
Yu Zhao302b4212009-05-18 13:51:32 +08008 * Address Translation Service 1.0
Yu Zhaod1b054d2009-03-20 11:25:11 +08009 */
10
11#include <linux/pci.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090012#include <linux/slab.h>
Yu Zhaod1b054d2009-03-20 11:25:11 +080013#include <linux/mutex.h>
Paul Gortmaker363c75d2011-05-27 09:37:25 -040014#include <linux/export.h>
Yu Zhaod1b054d2009-03-20 11:25:11 +080015#include <linux/string.h>
16#include <linux/delay.h>
Joerg Roedel5cdede22011-04-04 15:55:18 +020017#include <linux/pci-ats.h>
Yu Zhaod1b054d2009-03-20 11:25:11 +080018#include "pci.h"
19
Yu Zhaodd7cc442009-03-20 11:25:15 +080020#define VIRTFN_ID_LEN 16
Yu Zhaod1b054d2009-03-20 11:25:11 +080021
Wei Yangb07579c2015-03-25 16:23:48 +080022int pci_iov_virtfn_bus(struct pci_dev *dev, int vf_id)
Yu Zhaoa28724b2009-03-20 11:25:13 +080023{
Wei Yangb07579c2015-03-25 16:23:48 +080024 if (!dev->is_physfn)
25 return -EINVAL;
Yu Zhaoa28724b2009-03-20 11:25:13 +080026 return dev->bus->number + ((dev->devfn + dev->sriov->offset +
Wei Yangb07579c2015-03-25 16:23:48 +080027 dev->sriov->stride * vf_id) >> 8);
Yu Zhaoa28724b2009-03-20 11:25:13 +080028}
29
Wei Yangb07579c2015-03-25 16:23:48 +080030int pci_iov_virtfn_devfn(struct pci_dev *dev, int vf_id)
Yu Zhaoa28724b2009-03-20 11:25:13 +080031{
Wei Yangb07579c2015-03-25 16:23:48 +080032 if (!dev->is_physfn)
33 return -EINVAL;
Yu Zhaoa28724b2009-03-20 11:25:13 +080034 return (dev->devfn + dev->sriov->offset +
Wei Yangb07579c2015-03-25 16:23:48 +080035 dev->sriov->stride * vf_id) & 0xff;
Yu Zhaoa28724b2009-03-20 11:25:13 +080036}
37
Wei Yangf59dca22015-03-25 16:23:46 +080038/*
39 * Per SR-IOV spec sec 3.3.10 and 3.3.11, First VF Offset and VF Stride may
40 * change when NumVFs changes.
41 *
42 * Update iov->offset and iov->stride when NumVFs is written.
43 */
44static inline void pci_iov_set_numvfs(struct pci_dev *dev, int nr_virtfn)
45{
46 struct pci_sriov *iov = dev->sriov;
47
48 pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, nr_virtfn);
49 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &iov->offset);
50 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &iov->stride);
51}
52
Wei Yang4449f072015-03-25 16:23:47 +080053/*
54 * The PF consumes one bus number. NumVFs, First VF Offset, and VF Stride
55 * determine how many additional bus numbers will be consumed by VFs.
56 *
Alexander Duyckea9a8852015-10-29 16:20:50 -050057 * Iterate over all valid NumVFs, validate offset and stride, and calculate
58 * the maximum number of bus numbers that could ever be required.
Wei Yang4449f072015-03-25 16:23:47 +080059 */
Alexander Duyckea9a8852015-10-29 16:20:50 -050060static int compute_max_vf_buses(struct pci_dev *dev)
Wei Yang4449f072015-03-25 16:23:47 +080061{
62 struct pci_sriov *iov = dev->sriov;
Alexander Duyckea9a8852015-10-29 16:20:50 -050063 int nr_virtfn, busnr, rc = 0;
Wei Yang4449f072015-03-25 16:23:47 +080064
Alexander Duyckea9a8852015-10-29 16:20:50 -050065 for (nr_virtfn = iov->total_VFs; nr_virtfn; nr_virtfn--) {
Wei Yang4449f072015-03-25 16:23:47 +080066 pci_iov_set_numvfs(dev, nr_virtfn);
Alexander Duyckea9a8852015-10-29 16:20:50 -050067 if (!iov->offset || (nr_virtfn > 1 && !iov->stride)) {
68 rc = -EIO;
69 goto out;
70 }
71
Wei Yangb07579c2015-03-25 16:23:48 +080072 busnr = pci_iov_virtfn_bus(dev, nr_virtfn - 1);
Alexander Duyckea9a8852015-10-29 16:20:50 -050073 if (busnr > iov->max_VF_buses)
74 iov->max_VF_buses = busnr;
Wei Yang4449f072015-03-25 16:23:47 +080075 }
76
Alexander Duyckea9a8852015-10-29 16:20:50 -050077out:
78 pci_iov_set_numvfs(dev, 0);
79 return rc;
Wei Yang4449f072015-03-25 16:23:47 +080080}
81
Yu Zhaodd7cc442009-03-20 11:25:15 +080082static struct pci_bus *virtfn_add_bus(struct pci_bus *bus, int busnr)
83{
Yu Zhaodd7cc442009-03-20 11:25:15 +080084 struct pci_bus *child;
85
86 if (bus->number == busnr)
87 return bus;
88
89 child = pci_find_bus(pci_domain_nr(bus), busnr);
90 if (child)
91 return child;
92
93 child = pci_add_new_bus(bus, NULL, busnr);
94 if (!child)
95 return NULL;
96
Yinghai Lub7eac052012-05-17 18:51:13 -070097 pci_bus_insert_busn_res(child, busnr, busnr);
Yu Zhaodd7cc442009-03-20 11:25:15 +080098
99 return child;
100}
101
Jiang Liudc087f22013-05-25 21:48:37 +0800102static void virtfn_remove_bus(struct pci_bus *physbus, struct pci_bus *virtbus)
Yu Zhaodd7cc442009-03-20 11:25:15 +0800103{
Jiang Liudc087f22013-05-25 21:48:37 +0800104 if (physbus != virtbus && list_empty(&virtbus->devices))
105 pci_remove_bus(virtbus);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800106}
107
Wei Yang0e6c9122015-03-25 16:23:44 +0800108resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno)
109{
110 if (!dev->is_physfn)
111 return 0;
112
113 return dev->sriov->barsz[resno - PCI_IOV_RESOURCES];
114}
115
Wei Yangc194f7e2016-03-04 10:53:03 +1100116int pci_iov_add_virtfn(struct pci_dev *dev, int id, int reset)
Yu Zhaodd7cc442009-03-20 11:25:15 +0800117{
118 int i;
Jiang Liudc087f22013-05-25 21:48:37 +0800119 int rc = -ENOMEM;
Yu Zhaodd7cc442009-03-20 11:25:15 +0800120 u64 size;
121 char buf[VIRTFN_ID_LEN];
122 struct pci_dev *virtfn;
123 struct resource *res;
124 struct pci_sriov *iov = dev->sriov;
Gu Zheng8b1fce02013-05-25 21:48:31 +0800125 struct pci_bus *bus;
Yu Zhaodd7cc442009-03-20 11:25:15 +0800126
127 mutex_lock(&iov->dev->sriov->lock);
Wei Yangb07579c2015-03-25 16:23:48 +0800128 bus = virtfn_add_bus(dev->bus, pci_iov_virtfn_bus(dev, id));
Jiang Liudc087f22013-05-25 21:48:37 +0800129 if (!bus)
130 goto failed;
131
132 virtfn = pci_alloc_dev(bus);
133 if (!virtfn)
134 goto failed0;
135
Wei Yangb07579c2015-03-25 16:23:48 +0800136 virtfn->devfn = pci_iov_virtfn_devfn(dev, id);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800137 virtfn->vendor = dev->vendor;
138 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_DID, &virtfn->device);
Po Liu156c5532016-08-29 15:28:01 +0800139 rc = pci_setup_device(virtfn);
140 if (rc)
141 goto failed0;
142
Yu Zhaodd7cc442009-03-20 11:25:15 +0800143 virtfn->dev.parent = dev->dev.parent;
Xudong Haofbf33f52013-05-31 12:21:29 +0800144 virtfn->physfn = pci_dev_get(dev);
145 virtfn->is_virtfn = 1;
Alex Williamsonaa9319772014-01-09 08:36:08 -0700146 virtfn->multifunction = 0;
Yu Zhaodd7cc442009-03-20 11:25:15 +0800147
148 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
Bjorn Helgaasc1fe1f92015-03-25 16:23:45 +0800149 res = &dev->resource[i + PCI_IOV_RESOURCES];
Yu Zhaodd7cc442009-03-20 11:25:15 +0800150 if (!res->parent)
151 continue;
152 virtfn->resource[i].name = pci_name(virtfn);
153 virtfn->resource[i].flags = res->flags;
Wei Yang0e6c9122015-03-25 16:23:44 +0800154 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800155 virtfn->resource[i].start = res->start + size * id;
156 virtfn->resource[i].end = virtfn->resource[i].start + size - 1;
157 rc = request_resource(res, &virtfn->resource[i]);
158 BUG_ON(rc);
159 }
160
161 if (reset)
Yu Zhao8c1c6992009-06-13 15:52:13 +0800162 __pci_reset_function(virtfn);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800163
164 pci_device_add(virtfn, virtfn->bus);
165 mutex_unlock(&iov->dev->sriov->lock);
166
Yijing Wangc893d132014-05-30 11:01:03 +0800167 pci_bus_add_device(virtfn);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800168 sprintf(buf, "virtfn%u", id);
169 rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf);
170 if (rc)
171 goto failed1;
172 rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn");
173 if (rc)
174 goto failed2;
175
176 kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE);
177
178 return 0;
179
180failed2:
181 sysfs_remove_link(&dev->dev.kobj, buf);
182failed1:
183 pci_dev_put(dev);
184 mutex_lock(&iov->dev->sriov->lock);
Yinghai Lu210647a2012-02-25 13:54:20 -0800185 pci_stop_and_remove_bus_device(virtfn);
Jiang Liudc087f22013-05-25 21:48:37 +0800186failed0:
187 virtfn_remove_bus(dev->bus, bus);
188failed:
Yu Zhaodd7cc442009-03-20 11:25:15 +0800189 mutex_unlock(&iov->dev->sriov->lock);
190
191 return rc;
192}
193
Wei Yangc194f7e2016-03-04 10:53:03 +1100194void pci_iov_remove_virtfn(struct pci_dev *dev, int id, int reset)
Yu Zhaodd7cc442009-03-20 11:25:15 +0800195{
196 char buf[VIRTFN_ID_LEN];
Yu Zhaodd7cc442009-03-20 11:25:15 +0800197 struct pci_dev *virtfn;
198 struct pci_sriov *iov = dev->sriov;
199
Jiang Liudc087f22013-05-25 21:48:37 +0800200 virtfn = pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus),
Wei Yangb07579c2015-03-25 16:23:48 +0800201 pci_iov_virtfn_bus(dev, id),
202 pci_iov_virtfn_devfn(dev, id));
Yu Zhaodd7cc442009-03-20 11:25:15 +0800203 if (!virtfn)
204 return;
205
Yu Zhaodd7cc442009-03-20 11:25:15 +0800206 if (reset) {
207 device_release_driver(&virtfn->dev);
Yu Zhao8c1c6992009-06-13 15:52:13 +0800208 __pci_reset_function(virtfn);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800209 }
210
211 sprintf(buf, "virtfn%u", id);
212 sysfs_remove_link(&dev->dev.kobj, buf);
Yinghai Lu09cedbe2012-02-04 22:55:01 -0800213 /*
214 * pci_stop_dev() could have been called for this virtfn already,
215 * so the directory for the virtfn may have been removed before.
216 * Double check to avoid spurious sysfs warnings.
217 */
218 if (virtfn->dev.kobj.sd)
219 sysfs_remove_link(&virtfn->dev.kobj, "physfn");
Yu Zhaodd7cc442009-03-20 11:25:15 +0800220
221 mutex_lock(&iov->dev->sriov->lock);
Yinghai Lu210647a2012-02-25 13:54:20 -0800222 pci_stop_and_remove_bus_device(virtfn);
Jiang Liudc087f22013-05-25 21:48:37 +0800223 virtfn_remove_bus(dev->bus, virtfn->bus);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800224 mutex_unlock(&iov->dev->sriov->lock);
225
Jiang Liudc087f22013-05-25 21:48:37 +0800226 /* balance pci_get_domain_bus_and_slot() */
227 pci_dev_put(virtfn);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800228 pci_dev_put(dev);
229}
230
Wei Yang995df522015-03-25 16:23:49 +0800231int __weak pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
232{
Alexander Duycka39e3fc2015-10-29 16:21:11 -0500233 return 0;
234}
235
236int __weak pcibios_sriov_disable(struct pci_dev *pdev)
237{
238 return 0;
Wei Yang995df522015-03-25 16:23:49 +0800239}
240
Yu Zhaodd7cc442009-03-20 11:25:15 +0800241static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
242{
243 int rc;
Alexander Duyck3443c382015-10-29 16:21:05 -0500244 int i;
Yu Zhaodd7cc442009-03-20 11:25:15 +0800245 int nres;
Alexander Duyckce288ec2015-10-29 16:20:57 -0500246 u16 initial;
Yu Zhaodd7cc442009-03-20 11:25:15 +0800247 struct resource *res;
248 struct pci_dev *pdev;
249 struct pci_sriov *iov = dev->sriov;
Ram Paibbef98a2011-11-06 10:33:10 +0800250 int bars = 0;
Wei Yangb07579c2015-03-25 16:23:48 +0800251 int bus;
Yu Zhaodd7cc442009-03-20 11:25:15 +0800252
253 if (!nr_virtfn)
254 return 0;
255
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700256 if (iov->num_VFs)
Yu Zhaodd7cc442009-03-20 11:25:15 +0800257 return -EINVAL;
258
259 pci_read_config_word(dev, iov->pos + PCI_SRIOV_INITIAL_VF, &initial);
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700260 if (initial > iov->total_VFs ||
261 (!(iov->cap & PCI_SRIOV_CAP_VFM) && (initial != iov->total_VFs)))
Yu Zhaodd7cc442009-03-20 11:25:15 +0800262 return -EIO;
263
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700264 if (nr_virtfn < 0 || nr_virtfn > iov->total_VFs ||
Yu Zhaodd7cc442009-03-20 11:25:15 +0800265 (!(iov->cap & PCI_SRIOV_CAP_VFM) && (nr_virtfn > initial)))
266 return -EINVAL;
267
Yu Zhaodd7cc442009-03-20 11:25:15 +0800268 nres = 0;
269 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
Ram Paibbef98a2011-11-06 10:33:10 +0800270 bars |= (1 << (i + PCI_IOV_RESOURCES));
Bjorn Helgaasc1fe1f92015-03-25 16:23:45 +0800271 res = &dev->resource[i + PCI_IOV_RESOURCES];
Yu Zhaodd7cc442009-03-20 11:25:15 +0800272 if (res->parent)
273 nres++;
274 }
275 if (nres != iov->nres) {
276 dev_err(&dev->dev, "not enough MMIO resources for SR-IOV\n");
277 return -ENOMEM;
278 }
279
Wei Yangb07579c2015-03-25 16:23:48 +0800280 bus = pci_iov_virtfn_bus(dev, nr_virtfn - 1);
Bjorn Helgaas68f8e9f2015-03-25 16:23:42 +0800281 if (bus > dev->bus->busn_res.end) {
282 dev_err(&dev->dev, "can't enable %d VFs (bus %02x out of range of %pR)\n",
283 nr_virtfn, bus, &dev->bus->busn_res);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800284 return -ENOMEM;
285 }
286
Ram Paibbef98a2011-11-06 10:33:10 +0800287 if (pci_enable_resources(dev, bars)) {
288 dev_err(&dev->dev, "SR-IOV: IOV BARS not allocated\n");
289 return -ENOMEM;
290 }
291
Yu Zhaodd7cc442009-03-20 11:25:15 +0800292 if (iov->link != dev->devfn) {
293 pdev = pci_get_slot(dev->bus, iov->link);
294 if (!pdev)
295 return -ENODEV;
296
Jiang Liudc087f22013-05-25 21:48:37 +0800297 if (!pdev->is_physfn) {
298 pci_dev_put(pdev);
Stefan Assmann652d1102013-07-31 16:47:56 -0600299 return -ENOSYS;
Jiang Liudc087f22013-05-25 21:48:37 +0800300 }
Yu Zhaodd7cc442009-03-20 11:25:15 +0800301
302 rc = sysfs_create_link(&dev->dev.kobj,
303 &pdev->dev.kobj, "dep_link");
Jiang Liudc087f22013-05-25 21:48:37 +0800304 pci_dev_put(pdev);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800305 if (rc)
306 return rc;
307 }
308
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700309 iov->initial_VFs = initial;
Yu Zhaodd7cc442009-03-20 11:25:15 +0800310 if (nr_virtfn < initial)
311 initial = nr_virtfn;
312
Alexander Duyckc23b6132015-10-29 16:21:20 -0500313 rc = pcibios_sriov_enable(dev, initial);
314 if (rc) {
315 dev_err(&dev->dev, "failure %d from pcibios_sriov_enable()\n", rc);
316 goto err_pcibios;
Wei Yang995df522015-03-25 16:23:49 +0800317 }
318
Gavin Shanfb7c5212017-03-17 00:48:25 +0000319 pci_iov_set_numvfs(dev, nr_virtfn);
320 iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE;
321 pci_cfg_access_lock(dev);
322 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
323 msleep(100);
324 pci_cfg_access_unlock(dev);
325
Yu Zhaodd7cc442009-03-20 11:25:15 +0800326 for (i = 0; i < initial; i++) {
Wei Yangc194f7e2016-03-04 10:53:03 +1100327 rc = pci_iov_add_virtfn(dev, i, 0);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800328 if (rc)
329 goto failed;
330 }
331
332 kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE);
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700333 iov->num_VFs = nr_virtfn;
Yu Zhaodd7cc442009-03-20 11:25:15 +0800334
335 return 0;
336
337failed:
Alexander Duyck3443c382015-10-29 16:21:05 -0500338 while (i--)
Wei Yangc194f7e2016-03-04 10:53:03 +1100339 pci_iov_remove_virtfn(dev, i, 0);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800340
Alexander Duyckc23b6132015-10-29 16:21:20 -0500341 pcibios_sriov_disable(dev);
342err_pcibios:
Yu Zhaodd7cc442009-03-20 11:25:15 +0800343 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
Jan Kiszkafb51ccb2011-11-04 09:45:59 +0100344 pci_cfg_access_lock(dev);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800345 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
346 ssleep(1);
Jan Kiszkafb51ccb2011-11-04 09:45:59 +0100347 pci_cfg_access_unlock(dev);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800348
349 if (iov->link != dev->devfn)
350 sysfs_remove_link(&dev->dev.kobj, "dep_link");
351
Alexander Duyckb3908642015-10-29 16:21:16 -0500352 pci_iov_set_numvfs(dev, 0);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800353 return rc;
354}
355
356static void sriov_disable(struct pci_dev *dev)
357{
358 int i;
359 struct pci_sriov *iov = dev->sriov;
360
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700361 if (!iov->num_VFs)
Yu Zhaodd7cc442009-03-20 11:25:15 +0800362 return;
363
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700364 for (i = 0; i < iov->num_VFs; i++)
Wei Yangc194f7e2016-03-04 10:53:03 +1100365 pci_iov_remove_virtfn(dev, i, 0);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800366
Wei Yang995df522015-03-25 16:23:49 +0800367 pcibios_sriov_disable(dev);
368
Yu Zhaodd7cc442009-03-20 11:25:15 +0800369 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
Jan Kiszkafb51ccb2011-11-04 09:45:59 +0100370 pci_cfg_access_lock(dev);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800371 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
372 ssleep(1);
Jan Kiszkafb51ccb2011-11-04 09:45:59 +0100373 pci_cfg_access_unlock(dev);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800374
375 if (iov->link != dev->devfn)
376 sysfs_remove_link(&dev->dev.kobj, "dep_link");
377
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700378 iov->num_VFs = 0;
Wei Yangf59dca22015-03-25 16:23:46 +0800379 pci_iov_set_numvfs(dev, 0);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800380}
381
Yu Zhaod1b054d2009-03-20 11:25:11 +0800382static int sriov_init(struct pci_dev *dev, int pos)
383{
Wei Yang0e6c9122015-03-25 16:23:44 +0800384 int i, bar64;
Yu Zhaod1b054d2009-03-20 11:25:11 +0800385 int rc;
386 int nres;
387 u32 pgsz;
Alexander Duyckea9a8852015-10-29 16:20:50 -0500388 u16 ctrl, total;
Yu Zhaod1b054d2009-03-20 11:25:11 +0800389 struct pci_sriov *iov;
390 struct resource *res;
391 struct pci_dev *pdev;
392
Yu Zhaod1b054d2009-03-20 11:25:11 +0800393 pci_read_config_word(dev, pos + PCI_SRIOV_CTRL, &ctrl);
394 if (ctrl & PCI_SRIOV_CTRL_VFE) {
395 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, 0);
396 ssleep(1);
397 }
398
Yu Zhaod1b054d2009-03-20 11:25:11 +0800399 ctrl = 0;
400 list_for_each_entry(pdev, &dev->bus->devices, bus_list)
401 if (pdev->is_physfn)
402 goto found;
403
404 pdev = NULL;
405 if (pci_ari_enabled(dev->bus))
406 ctrl |= PCI_SRIOV_CTRL_ARI;
407
408found:
409 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, ctrl);
Yu Zhaod1b054d2009-03-20 11:25:11 +0800410
Ben Sheltonff45f9d2015-10-29 16:20:31 -0500411 pci_read_config_word(dev, pos + PCI_SRIOV_TOTAL_VF, &total);
412 if (!total)
413 return 0;
Yu Zhaod1b054d2009-03-20 11:25:11 +0800414
415 pci_read_config_dword(dev, pos + PCI_SRIOV_SUP_PGSIZE, &pgsz);
416 i = PAGE_SHIFT > 12 ? PAGE_SHIFT - 12 : 0;
417 pgsz &= ~((1 << i) - 1);
418 if (!pgsz)
419 return -EIO;
420
421 pgsz &= ~(pgsz - 1);
Vaidyanathan Srinivasan8161fe92012-02-02 23:11:20 +0530422 pci_write_config_dword(dev, pos + PCI_SRIOV_SYS_PGSIZE, pgsz);
Yu Zhaod1b054d2009-03-20 11:25:11 +0800423
Wei Yang0e6c9122015-03-25 16:23:44 +0800424 iov = kzalloc(sizeof(*iov), GFP_KERNEL);
425 if (!iov)
426 return -ENOMEM;
427
Yu Zhaod1b054d2009-03-20 11:25:11 +0800428 nres = 0;
429 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
Bjorn Helgaasc1fe1f92015-03-25 16:23:45 +0800430 res = &dev->resource[i + PCI_IOV_RESOURCES];
David Daney11183992015-10-29 17:35:40 -0500431 /*
432 * If it is already FIXED, don't change it, something
433 * (perhaps EA or header fixups) wants it this way.
434 */
435 if (res->flags & IORESOURCE_PCI_FIXED)
436 bar64 = (res->flags & IORESOURCE_MEM_64) ? 1 : 0;
437 else
438 bar64 = __pci_read_base(dev, pci_bar_unknown, res,
439 pos + PCI_SRIOV_BAR + i * 4);
Yu Zhaod1b054d2009-03-20 11:25:11 +0800440 if (!res->flags)
441 continue;
442 if (resource_size(res) & (PAGE_SIZE - 1)) {
443 rc = -EIO;
444 goto failed;
445 }
Wei Yang0e6c9122015-03-25 16:23:44 +0800446 iov->barsz[i] = resource_size(res);
Yu Zhaod1b054d2009-03-20 11:25:11 +0800447 res->end = res->start + resource_size(res) * total - 1;
Wei Yange88ae012015-03-25 16:23:43 +0800448 dev_info(&dev->dev, "VF(n) BAR%d space: %pR (contains BAR%d for %d VFs)\n",
449 i, res, i, total);
Wei Yang0e6c9122015-03-25 16:23:44 +0800450 i += bar64;
Yu Zhaod1b054d2009-03-20 11:25:11 +0800451 nres++;
452 }
453
Yu Zhaod1b054d2009-03-20 11:25:11 +0800454 iov->pos = pos;
455 iov->nres = nres;
456 iov->ctrl = ctrl;
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700457 iov->total_VFs = total;
Yu Zhaod1b054d2009-03-20 11:25:11 +0800458 iov->pgsz = pgsz;
459 iov->self = dev;
460 pci_read_config_dword(dev, pos + PCI_SRIOV_CAP, &iov->cap);
461 pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link);
Yijing Wang62f87c02012-07-24 17:20:03 +0800462 if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END)
Yu Zhao4d135db2009-05-20 17:11:57 +0800463 iov->link = PCI_DEVFN(PCI_SLOT(dev->devfn), iov->link);
Yu Zhaod1b054d2009-03-20 11:25:11 +0800464
465 if (pdev)
466 iov->dev = pci_dev_get(pdev);
Yu Zhaoe277d2f2009-05-18 13:51:33 +0800467 else
Yu Zhaod1b054d2009-03-20 11:25:11 +0800468 iov->dev = dev;
Yu Zhaoe277d2f2009-05-18 13:51:33 +0800469
470 mutex_init(&iov->lock);
Yu Zhaod1b054d2009-03-20 11:25:11 +0800471
472 dev->sriov = iov;
473 dev->is_physfn = 1;
Alexander Duyckea9a8852015-10-29 16:20:50 -0500474 rc = compute_max_vf_buses(dev);
475 if (rc)
476 goto fail_max_buses;
Yu Zhaod1b054d2009-03-20 11:25:11 +0800477
478 return 0;
479
Alexander Duyckea9a8852015-10-29 16:20:50 -0500480fail_max_buses:
481 dev->sriov = NULL;
482 dev->is_physfn = 0;
Yu Zhaod1b054d2009-03-20 11:25:11 +0800483failed:
484 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
Bjorn Helgaasc1fe1f92015-03-25 16:23:45 +0800485 res = &dev->resource[i + PCI_IOV_RESOURCES];
Yu Zhaod1b054d2009-03-20 11:25:11 +0800486 res->flags = 0;
487 }
488
Wei Yang0e6c9122015-03-25 16:23:44 +0800489 kfree(iov);
Yu Zhaod1b054d2009-03-20 11:25:11 +0800490 return rc;
491}
492
493static void sriov_release(struct pci_dev *dev)
494{
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700495 BUG_ON(dev->sriov->num_VFs);
Yu Zhaodd7cc442009-03-20 11:25:15 +0800496
Yu Zhaoe277d2f2009-05-18 13:51:33 +0800497 if (dev != dev->sriov->dev)
Yu Zhaod1b054d2009-03-20 11:25:11 +0800498 pci_dev_put(dev->sriov->dev);
499
Yu Zhaoe277d2f2009-05-18 13:51:33 +0800500 mutex_destroy(&dev->sriov->lock);
501
Yu Zhaod1b054d2009-03-20 11:25:11 +0800502 kfree(dev->sriov);
503 dev->sriov = NULL;
504}
505
Yu Zhao8c5cdb62009-03-20 11:25:12 +0800506static void sriov_restore_state(struct pci_dev *dev)
507{
508 int i;
509 u16 ctrl;
510 struct pci_sriov *iov = dev->sriov;
511
512 pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &ctrl);
513 if (ctrl & PCI_SRIOV_CTRL_VFE)
514 return;
515
516 for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++)
517 pci_update_resource(dev, i);
518
519 pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz);
Wei Yangf59dca22015-03-25 16:23:46 +0800520 pci_iov_set_numvfs(dev, iov->num_VFs);
Yu Zhao8c5cdb62009-03-20 11:25:12 +0800521 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
522 if (iov->ctrl & PCI_SRIOV_CTRL_VFE)
523 msleep(100);
524}
525
Yu Zhaod1b054d2009-03-20 11:25:11 +0800526/**
527 * pci_iov_init - initialize the IOV capability
528 * @dev: the PCI device
529 *
530 * Returns 0 on success, or negative on failure.
531 */
532int pci_iov_init(struct pci_dev *dev)
533{
534 int pos;
535
Kenji Kaneshige5f4d91a2009-11-11 14:36:17 +0900536 if (!pci_is_pcie(dev))
Yu Zhaod1b054d2009-03-20 11:25:11 +0800537 return -ENODEV;
538
539 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV);
540 if (pos)
541 return sriov_init(dev, pos);
542
543 return -ENODEV;
544}
545
546/**
547 * pci_iov_release - release resources used by the IOV capability
548 * @dev: the PCI device
549 */
550void pci_iov_release(struct pci_dev *dev)
551{
552 if (dev->is_physfn)
553 sriov_release(dev);
554}
555
556/**
Bjorn Helgaas6a5f3e62017-03-17 00:48:22 +0000557 * pci_iov_update_resource - update a VF BAR
558 * @dev: the PCI device
559 * @resno: the resource number
560 *
561 * Update a VF BAR in the SR-IOV capability of a PF.
562 */
563void pci_iov_update_resource(struct pci_dev *dev, int resno)
564{
565 struct pci_sriov *iov = dev->is_physfn ? dev->sriov : NULL;
566 struct resource *res = dev->resource + resno;
567 int vf_bar = resno - PCI_IOV_RESOURCES;
568 struct pci_bus_region region;
Bjorn Helgaasa38012d2017-03-17 00:48:24 +0000569 u16 cmd;
Bjorn Helgaas6a5f3e62017-03-17 00:48:22 +0000570 u32 new;
571 int reg;
572
573 /*
574 * The generic pci_restore_bars() path calls this for all devices,
575 * including VFs and non-SR-IOV devices. If this is not a PF, we
576 * have nothing to do.
577 */
578 if (!iov)
579 return;
580
Bjorn Helgaasa38012d2017-03-17 00:48:24 +0000581 pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &cmd);
582 if ((cmd & PCI_SRIOV_CTRL_VFE) && (cmd & PCI_SRIOV_CTRL_MSE)) {
583 dev_WARN(&dev->dev, "can't update enabled VF BAR%d %pR\n",
584 vf_bar, res);
585 return;
586 }
587
Bjorn Helgaas6a5f3e62017-03-17 00:48:22 +0000588 /*
589 * Ignore unimplemented BARs, unused resource slots for 64-bit
590 * BARs, and non-movable resources, e.g., those described via
591 * Enhanced Allocation.
592 */
593 if (!res->flags)
594 return;
595
596 if (res->flags & IORESOURCE_UNSET)
597 return;
598
599 if (res->flags & IORESOURCE_PCI_FIXED)
600 return;
601
602 pcibios_resource_to_bus(dev->bus, &region, res);
603 new = region.start;
604 new |= res->flags & ~PCI_BASE_ADDRESS_MEM_MASK;
605
606 reg = iov->pos + PCI_SRIOV_BAR + 4 * vf_bar;
607 pci_write_config_dword(dev, reg, new);
608 if (res->flags & IORESOURCE_MEM_64) {
609 new = region.start >> 16 >> 16;
610 pci_write_config_dword(dev, reg + 4, new);
611 }
612}
613
Wei Yang978d2d62015-03-25 16:23:50 +0800614resource_size_t __weak pcibios_iov_resource_alignment(struct pci_dev *dev,
615 int resno)
616{
617 return pci_iov_resource_size(dev, resno);
618}
619
Yu Zhao8c5cdb62009-03-20 11:25:12 +0800620/**
Chris Wright6faf17f2009-08-28 13:00:06 -0700621 * pci_sriov_resource_alignment - get resource alignment for VF BAR
622 * @dev: the PCI device
623 * @resno: the resource number
624 *
625 * Returns the alignment of the VF BAR found in the SR-IOV capability.
626 * This is not the same as the resource size which is defined as
627 * the VF BAR size multiplied by the number of VFs. The alignment
628 * is just the VF BAR size.
629 */
Cam Macdonell0e522472010-09-07 17:25:20 -0700630resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno)
Chris Wright6faf17f2009-08-28 13:00:06 -0700631{
Wei Yang978d2d62015-03-25 16:23:50 +0800632 return pcibios_iov_resource_alignment(dev, resno);
Chris Wright6faf17f2009-08-28 13:00:06 -0700633}
634
635/**
Yu Zhao8c5cdb62009-03-20 11:25:12 +0800636 * pci_restore_iov_state - restore the state of the IOV capability
637 * @dev: the PCI device
638 */
639void pci_restore_iov_state(struct pci_dev *dev)
640{
641 if (dev->is_physfn)
642 sriov_restore_state(dev);
643}
Yu Zhaoa28724b2009-03-20 11:25:13 +0800644
645/**
646 * pci_iov_bus_range - find bus range used by Virtual Function
647 * @bus: the PCI bus
648 *
649 * Returns max number of buses (exclude current one) used by Virtual
650 * Functions.
651 */
652int pci_iov_bus_range(struct pci_bus *bus)
653{
654 int max = 0;
Yu Zhaoa28724b2009-03-20 11:25:13 +0800655 struct pci_dev *dev;
656
657 list_for_each_entry(dev, &bus->devices, bus_list) {
658 if (!dev->is_physfn)
659 continue;
Wei Yang4449f072015-03-25 16:23:47 +0800660 if (dev->sriov->max_VF_buses > max)
661 max = dev->sriov->max_VF_buses;
Yu Zhaoa28724b2009-03-20 11:25:13 +0800662 }
663
664 return max ? max - bus->number : 0;
665}
Yu Zhaodd7cc442009-03-20 11:25:15 +0800666
667/**
668 * pci_enable_sriov - enable the SR-IOV capability
669 * @dev: the PCI device
Randy Dunlap52a88732009-04-01 17:45:30 -0700670 * @nr_virtfn: number of virtual functions to enable
Yu Zhaodd7cc442009-03-20 11:25:15 +0800671 *
672 * Returns 0 on success, or negative on failure.
673 */
674int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn)
675{
676 might_sleep();
677
678 if (!dev->is_physfn)
Stefan Assmann652d1102013-07-31 16:47:56 -0600679 return -ENOSYS;
Yu Zhaodd7cc442009-03-20 11:25:15 +0800680
681 return sriov_enable(dev, nr_virtfn);
682}
683EXPORT_SYMBOL_GPL(pci_enable_sriov);
684
685/**
686 * pci_disable_sriov - disable the SR-IOV capability
687 * @dev: the PCI device
688 */
689void pci_disable_sriov(struct pci_dev *dev)
690{
691 might_sleep();
692
693 if (!dev->is_physfn)
694 return;
695
696 sriov_disable(dev);
697}
698EXPORT_SYMBOL_GPL(pci_disable_sriov);
Yu Zhao74bb1bc2009-03-20 11:25:16 +0800699
700/**
Williams, Mitch Afb8a0d92010-02-10 01:43:04 +0000701 * pci_num_vf - return number of VFs associated with a PF device_release_driver
702 * @dev: the PCI device
703 *
704 * Returns number of VFs, or 0 if SR-IOV is not enabled.
705 */
706int pci_num_vf(struct pci_dev *dev)
707{
Bjorn Helgaas1452cd72012-11-09 20:35:01 -0700708 if (!dev->is_physfn)
Williams, Mitch Afb8a0d92010-02-10 01:43:04 +0000709 return 0;
Bjorn Helgaas1452cd72012-11-09 20:35:01 -0700710
711 return dev->sriov->num_VFs;
Williams, Mitch Afb8a0d92010-02-10 01:43:04 +0000712}
713EXPORT_SYMBOL_GPL(pci_num_vf);
Donald Dutilebff73152012-11-05 15:20:37 -0500714
715/**
Alexander Duyck5a8eb242013-04-25 04:42:29 +0000716 * pci_vfs_assigned - returns number of VFs are assigned to a guest
717 * @dev: the PCI device
718 *
719 * Returns number of VFs belonging to this device that are assigned to a guest.
Stefan Assmann652d1102013-07-31 16:47:56 -0600720 * If device is not a physical function returns 0.
Alexander Duyck5a8eb242013-04-25 04:42:29 +0000721 */
722int pci_vfs_assigned(struct pci_dev *dev)
723{
724 struct pci_dev *vfdev;
725 unsigned int vfs_assigned = 0;
726 unsigned short dev_id;
727
728 /* only search if we are a PF */
729 if (!dev->is_physfn)
730 return 0;
731
732 /*
733 * determine the device ID for the VFs, the vendor ID will be the
734 * same as the PF so there is no need to check for that one
735 */
736 pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_VF_DID, &dev_id);
737
738 /* loop through all the VFs to see if we own any that are assigned */
739 vfdev = pci_get_device(dev->vendor, dev_id, NULL);
740 while (vfdev) {
741 /*
742 * It is considered assigned if it is a virtual function with
743 * our dev as the physical function and the assigned bit is set
744 */
745 if (vfdev->is_virtfn && (vfdev->physfn == dev) &&
Ethan Zhaobe634972014-09-09 10:21:28 +0800746 pci_is_dev_assigned(vfdev))
Alexander Duyck5a8eb242013-04-25 04:42:29 +0000747 vfs_assigned++;
748
749 vfdev = pci_get_device(dev->vendor, dev_id, vfdev);
750 }
751
752 return vfs_assigned;
753}
754EXPORT_SYMBOL_GPL(pci_vfs_assigned);
755
756/**
Donald Dutilebff73152012-11-05 15:20:37 -0500757 * pci_sriov_set_totalvfs -- reduce the TotalVFs available
758 * @dev: the PCI PF device
Randy Dunlap2094f162013-01-09 17:12:52 -0800759 * @numvfs: number that should be used for TotalVFs supported
Donald Dutilebff73152012-11-05 15:20:37 -0500760 *
761 * Should be called from PF driver's probe routine with
762 * device's mutex held.
763 *
764 * Returns 0 if PF is an SRIOV-capable device and
Stefan Assmann652d1102013-07-31 16:47:56 -0600765 * value of numvfs valid. If not a PF return -ENOSYS;
766 * if numvfs is invalid return -EINVAL;
Donald Dutilebff73152012-11-05 15:20:37 -0500767 * if VFs already enabled, return -EBUSY.
768 */
769int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs)
770{
Stefan Assmann652d1102013-07-31 16:47:56 -0600771 if (!dev->is_physfn)
772 return -ENOSYS;
773 if (numvfs > dev->sriov->total_VFs)
Donald Dutilebff73152012-11-05 15:20:37 -0500774 return -EINVAL;
775
776 /* Shouldn't change if VFs already enabled */
777 if (dev->sriov->ctrl & PCI_SRIOV_CTRL_VFE)
778 return -EBUSY;
779 else
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700780 dev->sriov->driver_max_VFs = numvfs;
Donald Dutilebff73152012-11-05 15:20:37 -0500781
782 return 0;
783}
784EXPORT_SYMBOL_GPL(pci_sriov_set_totalvfs);
785
786/**
Jonghwan Choiddc191f2013-07-08 14:02:43 -0600787 * pci_sriov_get_totalvfs -- get total VFs supported on this device
Donald Dutilebff73152012-11-05 15:20:37 -0500788 * @dev: the PCI PF device
789 *
790 * For a PCIe device with SRIOV support, return the PCIe
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700791 * SRIOV capability value of TotalVFs or the value of driver_max_VFs
Stefan Assmann652d1102013-07-31 16:47:56 -0600792 * if the driver reduced it. Otherwise 0.
Donald Dutilebff73152012-11-05 15:20:37 -0500793 */
794int pci_sriov_get_totalvfs(struct pci_dev *dev)
795{
Bjorn Helgaas1452cd72012-11-09 20:35:01 -0700796 if (!dev->is_physfn)
Stefan Assmann652d1102013-07-31 16:47:56 -0600797 return 0;
Donald Dutilebff73152012-11-05 15:20:37 -0500798
Bjorn Helgaas6b136722012-11-09 20:27:53 -0700799 if (dev->sriov->driver_max_VFs)
800 return dev->sriov->driver_max_VFs;
Bjorn Helgaas1452cd72012-11-09 20:35:01 -0700801
802 return dev->sriov->total_VFs;
Donald Dutilebff73152012-11-05 15:20:37 -0500803}
804EXPORT_SYMBOL_GPL(pci_sriov_get_totalvfs);