blob: 9355f754c7c2a00b2bab83dd5c332e34e9c52cb9 [file] [log] [blame]
Joerg Roedeldb3c33c2011-09-27 15:57:13 +02001/*
2 * drivers/pci/ats.c
3 *
4 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
Joerg Roedelc320b972011-09-27 15:57:15 +02005 * Copyright (C) 2011 Advanced Micro Devices,
Joerg Roedeldb3c33c2011-09-27 15:57:13 +02006 *
7 * PCI Express I/O Virtualization (IOV) support.
8 * Address Translation Service 1.0
Joerg Roedelc320b972011-09-27 15:57:15 +02009 * Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com>
Joerg Roedel086ac112011-09-27 15:57:16 +020010 * PASID support added by Joerg Roedel <joerg.roedel@amd.com>
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020011 */
12
Paul Gortmaker363c75d2011-05-27 09:37:25 -040013#include <linux/export.h>
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020014#include <linux/pci-ats.h>
15#include <linux/pci.h>
James Bottomley8c451942011-11-29 19:20:23 +000016#include <linux/slab.h>
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020017
18#include "pci.h"
19
Bjorn Helgaasafdd5962015-07-17 15:35:18 -050020void pci_ats_init(struct pci_dev *dev)
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020021{
22 int pos;
23 u16 cap;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020024
25 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS);
26 if (!pos)
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050027 return;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020028
Bjorn Helgaasd544d752015-07-17 15:15:19 -050029 dev->ats_cap = pos;
30 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap);
31 dev->ats_qdep = PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) :
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020032 PCI_ATS_MAX_QDEP;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020033}
34
35/**
36 * pci_enable_ats - enable the ATS capability
37 * @dev: the PCI device
38 * @ps: the IOMMU page shift
39 *
40 * Returns 0 on success, or negative on failure.
41 */
42int pci_enable_ats(struct pci_dev *dev, int ps)
43{
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020044 u16 ctrl;
Bjorn Helgaasc39127d2015-07-17 15:38:13 -050045 struct pci_dev *pdev;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020046
Bjorn Helgaasd544d752015-07-17 15:15:19 -050047 if (!dev->ats_cap)
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050048 return -EINVAL;
49
Bjorn Helgaasa021f302015-07-17 15:43:27 -050050 if (WARN_ON(pci_ats_enabled(dev)))
51 return -EBUSY;
52
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020053 if (ps < PCI_ATS_MIN_STU)
54 return -EINVAL;
55
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050056 /*
57 * Note that enabling ATS on a VF fails unless it's already enabled
58 * with the same STU on the PF.
59 */
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020060 ctrl = PCI_ATS_CTRL_ENABLE;
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050061 if (dev->is_virtfn) {
Bjorn Helgaasc39127d2015-07-17 15:38:13 -050062 pdev = pci_physfn(dev);
Bjorn Helgaasd544d752015-07-17 15:15:19 -050063 if (pdev->ats_stu != ps)
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050064 return -EINVAL;
65
Bjorn Helgaasd544d752015-07-17 15:15:19 -050066 atomic_inc(&pdev->ats_ref_cnt); /* count enabled VFs */
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050067 } else {
Bjorn Helgaasd544d752015-07-17 15:15:19 -050068 dev->ats_stu = ps;
69 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050070 }
Bjorn Helgaasd544d752015-07-17 15:15:19 -050071 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020072
Bjorn Helgaasd544d752015-07-17 15:15:19 -050073 dev->ats_enabled = 1;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020074 return 0;
75}
Joerg Roedeld4c06362011-09-27 15:57:14 +020076EXPORT_SYMBOL_GPL(pci_enable_ats);
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020077
78/**
79 * pci_disable_ats - disable the ATS capability
80 * @dev: the PCI device
81 */
82void pci_disable_ats(struct pci_dev *dev)
83{
Bjorn Helgaasc39127d2015-07-17 15:38:13 -050084 struct pci_dev *pdev;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020085 u16 ctrl;
86
Bjorn Helgaasa021f302015-07-17 15:43:27 -050087 if (WARN_ON(!pci_ats_enabled(dev)))
88 return;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020089
Bjorn Helgaasd544d752015-07-17 15:15:19 -050090 if (atomic_read(&dev->ats_ref_cnt))
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050091 return; /* VFs still enabled */
92
93 if (dev->is_virtfn) {
Bjorn Helgaasc39127d2015-07-17 15:38:13 -050094 pdev = pci_physfn(dev);
Bjorn Helgaasd544d752015-07-17 15:15:19 -050095 atomic_dec(&pdev->ats_ref_cnt);
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050096 }
97
Bjorn Helgaasd544d752015-07-17 15:15:19 -050098 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl);
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020099 ctrl &= ~PCI_ATS_CTRL_ENABLE;
Bjorn Helgaasd544d752015-07-17 15:15:19 -0500100 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200101
Bjorn Helgaasd544d752015-07-17 15:15:19 -0500102 dev->ats_enabled = 0;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200103}
Joerg Roedeld4c06362011-09-27 15:57:14 +0200104EXPORT_SYMBOL_GPL(pci_disable_ats);
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200105
Hao, Xudong1900ca12011-12-17 21:24:40 +0800106void pci_restore_ats_state(struct pci_dev *dev)
107{
108 u16 ctrl;
109
110 if (!pci_ats_enabled(dev))
111 return;
Hao, Xudong1900ca12011-12-17 21:24:40 +0800112
113 ctrl = PCI_ATS_CTRL_ENABLE;
114 if (!dev->is_virtfn)
Bjorn Helgaasd544d752015-07-17 15:15:19 -0500115 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
116 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
Hao, Xudong1900ca12011-12-17 21:24:40 +0800117}
118EXPORT_SYMBOL_GPL(pci_restore_ats_state);
119
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200120/**
121 * pci_ats_queue_depth - query the ATS Invalidate Queue Depth
122 * @dev: the PCI device
123 *
124 * Returns the queue depth on success, or negative on failure.
125 *
126 * The ATS spec uses 0 in the Invalidate Queue Depth field to
127 * indicate that the function can accept 32 Invalidate Request.
128 * But here we use the `real' values (i.e. 1~32) for the Queue
129 * Depth; and 0 indicates the function shares the Queue with
130 * other functions (doesn't exclusively own a Queue).
131 */
132int pci_ats_queue_depth(struct pci_dev *dev)
133{
Bjorn Helgaas3c765392015-07-17 15:30:26 -0500134 if (!dev->ats_cap)
135 return -EINVAL;
136
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200137 if (dev->is_virtfn)
138 return 0;
139
Bjorn Helgaas3c765392015-07-17 15:30:26 -0500140 return dev->ats_qdep;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200141}
Joerg Roedeld4c06362011-09-27 15:57:14 +0200142EXPORT_SYMBOL_GPL(pci_ats_queue_depth);
Joerg Roedelc320b972011-09-27 15:57:15 +0200143
144#ifdef CONFIG_PCI_PRI
145/**
146 * pci_enable_pri - Enable PRI capability
147 * @ pdev: PCI device structure
148 *
149 * Returns 0 on success, negative value on error
150 */
151int pci_enable_pri(struct pci_dev *pdev, u32 reqs)
152{
153 u16 control, status;
154 u32 max_requests;
155 int pos;
156
Alex Williamson69166fb2011-11-02 14:07:15 -0600157 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
Joerg Roedelc320b972011-09-27 15:57:15 +0200158 if (!pos)
159 return -EINVAL;
160
Alex Williamson91f57d52011-11-11 10:07:36 -0700161 pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
162 pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status);
163 if ((control & PCI_PRI_CTRL_ENABLE) ||
164 !(status & PCI_PRI_STATUS_STOPPED))
Joerg Roedelc320b972011-09-27 15:57:15 +0200165 return -EBUSY;
166
Alex Williamson91f57d52011-11-11 10:07:36 -0700167 pci_read_config_dword(pdev, pos + PCI_PRI_MAX_REQ, &max_requests);
Joerg Roedelc320b972011-09-27 15:57:15 +0200168 reqs = min(max_requests, reqs);
Alex Williamson91f57d52011-11-11 10:07:36 -0700169 pci_write_config_dword(pdev, pos + PCI_PRI_ALLOC_REQ, reqs);
Joerg Roedelc320b972011-09-27 15:57:15 +0200170
Alex Williamson91f57d52011-11-11 10:07:36 -0700171 control |= PCI_PRI_CTRL_ENABLE;
172 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
Joerg Roedelc320b972011-09-27 15:57:15 +0200173
174 return 0;
175}
176EXPORT_SYMBOL_GPL(pci_enable_pri);
177
178/**
179 * pci_disable_pri - Disable PRI capability
180 * @pdev: PCI device structure
181 *
182 * Only clears the enabled-bit, regardless of its former value
183 */
184void pci_disable_pri(struct pci_dev *pdev)
185{
186 u16 control;
187 int pos;
188
Alex Williamson69166fb2011-11-02 14:07:15 -0600189 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
Joerg Roedelc320b972011-09-27 15:57:15 +0200190 if (!pos)
191 return;
192
Alex Williamson91f57d52011-11-11 10:07:36 -0700193 pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
194 control &= ~PCI_PRI_CTRL_ENABLE;
195 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
Joerg Roedelc320b972011-09-27 15:57:15 +0200196}
197EXPORT_SYMBOL_GPL(pci_disable_pri);
198
199/**
Joerg Roedelc320b972011-09-27 15:57:15 +0200200 * pci_reset_pri - Resets device's PRI state
201 * @pdev: PCI device structure
202 *
203 * The PRI capability must be disabled before this function is called.
204 * Returns 0 on success, negative value on error.
205 */
206int pci_reset_pri(struct pci_dev *pdev)
207{
208 u16 control;
209 int pos;
210
Alex Williamson69166fb2011-11-02 14:07:15 -0600211 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
Joerg Roedelc320b972011-09-27 15:57:15 +0200212 if (!pos)
213 return -EINVAL;
214
Alex Williamson91f57d52011-11-11 10:07:36 -0700215 pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
216 if (control & PCI_PRI_CTRL_ENABLE)
Joerg Roedelc320b972011-09-27 15:57:15 +0200217 return -EBUSY;
218
Alex Williamson91f57d52011-11-11 10:07:36 -0700219 control |= PCI_PRI_CTRL_RESET;
Joerg Roedelc320b972011-09-27 15:57:15 +0200220
Alex Williamson91f57d52011-11-11 10:07:36 -0700221 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
Joerg Roedelc320b972011-09-27 15:57:15 +0200222
223 return 0;
224}
225EXPORT_SYMBOL_GPL(pci_reset_pri);
Joerg Roedelc320b972011-09-27 15:57:15 +0200226#endif /* CONFIG_PCI_PRI */
Joerg Roedel086ac112011-09-27 15:57:16 +0200227
228#ifdef CONFIG_PCI_PASID
229/**
230 * pci_enable_pasid - Enable the PASID capability
231 * @pdev: PCI device structure
232 * @features: Features to enable
233 *
234 * Returns 0 on success, negative value on error. This function checks
235 * whether the features are actually supported by the device and returns
236 * an error if not.
237 */
238int pci_enable_pasid(struct pci_dev *pdev, int features)
239{
240 u16 control, supported;
241 int pos;
242
Alex Williamson69166fb2011-11-02 14:07:15 -0600243 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
Joerg Roedel086ac112011-09-27 15:57:16 +0200244 if (!pos)
245 return -EINVAL;
246
Alex Williamson91f57d52011-11-11 10:07:36 -0700247 pci_read_config_word(pdev, pos + PCI_PASID_CTRL, &control);
248 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported);
Joerg Roedel086ac112011-09-27 15:57:16 +0200249
Alex Williamson91f57d52011-11-11 10:07:36 -0700250 if (control & PCI_PASID_CTRL_ENABLE)
Joerg Roedel086ac112011-09-27 15:57:16 +0200251 return -EINVAL;
252
Alex Williamson91f57d52011-11-11 10:07:36 -0700253 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
Joerg Roedel086ac112011-09-27 15:57:16 +0200254
255 /* User wants to enable anything unsupported? */
256 if ((supported & features) != features)
257 return -EINVAL;
258
Alex Williamson91f57d52011-11-11 10:07:36 -0700259 control = PCI_PASID_CTRL_ENABLE | features;
Joerg Roedel086ac112011-09-27 15:57:16 +0200260
Alex Williamson91f57d52011-11-11 10:07:36 -0700261 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control);
Joerg Roedel086ac112011-09-27 15:57:16 +0200262
263 return 0;
264}
265EXPORT_SYMBOL_GPL(pci_enable_pasid);
266
267/**
268 * pci_disable_pasid - Disable the PASID capability
269 * @pdev: PCI device structure
270 *
271 */
272void pci_disable_pasid(struct pci_dev *pdev)
273{
274 u16 control = 0;
275 int pos;
276
Alex Williamson69166fb2011-11-02 14:07:15 -0600277 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
Joerg Roedel086ac112011-09-27 15:57:16 +0200278 if (!pos)
279 return;
280
Alex Williamson91f57d52011-11-11 10:07:36 -0700281 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control);
Joerg Roedel086ac112011-09-27 15:57:16 +0200282}
283EXPORT_SYMBOL_GPL(pci_disable_pasid);
284
285/**
286 * pci_pasid_features - Check which PASID features are supported
287 * @pdev: PCI device structure
288 *
289 * Returns a negative value when no PASI capability is present.
290 * Otherwise is returns a bitmask with supported features. Current
291 * features reported are:
Alex Williamson91f57d52011-11-11 10:07:36 -0700292 * PCI_PASID_CAP_EXEC - Execute permission supported
Bjorn Helgaasf7625982013-11-14 11:28:18 -0700293 * PCI_PASID_CAP_PRIV - Privileged mode supported
Joerg Roedel086ac112011-09-27 15:57:16 +0200294 */
295int pci_pasid_features(struct pci_dev *pdev)
296{
297 u16 supported;
298 int pos;
299
Alex Williamson69166fb2011-11-02 14:07:15 -0600300 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
Joerg Roedel086ac112011-09-27 15:57:16 +0200301 if (!pos)
302 return -EINVAL;
303
Alex Williamson91f57d52011-11-11 10:07:36 -0700304 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported);
Joerg Roedel086ac112011-09-27 15:57:16 +0200305
Alex Williamson91f57d52011-11-11 10:07:36 -0700306 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
Joerg Roedel086ac112011-09-27 15:57:16 +0200307
308 return supported;
309}
310EXPORT_SYMBOL_GPL(pci_pasid_features);
311
312#define PASID_NUMBER_SHIFT 8
313#define PASID_NUMBER_MASK (0x1f << PASID_NUMBER_SHIFT)
314/**
315 * pci_max_pasid - Get maximum number of PASIDs supported by device
316 * @pdev: PCI device structure
317 *
318 * Returns negative value when PASID capability is not present.
319 * Otherwise it returns the numer of supported PASIDs.
320 */
321int pci_max_pasids(struct pci_dev *pdev)
322{
323 u16 supported;
324 int pos;
325
Alex Williamson69166fb2011-11-02 14:07:15 -0600326 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
Joerg Roedel086ac112011-09-27 15:57:16 +0200327 if (!pos)
328 return -EINVAL;
329
Alex Williamson91f57d52011-11-11 10:07:36 -0700330 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported);
Joerg Roedel086ac112011-09-27 15:57:16 +0200331
332 supported = (supported & PASID_NUMBER_MASK) >> PASID_NUMBER_SHIFT;
333
334 return (1 << supported);
335}
336EXPORT_SYMBOL_GPL(pci_max_pasids);
337#endif /* CONFIG_PCI_PASID */