blob: 690ae6e6786c59c2a9dc0bbfb69e6feaa07be57e [file] [log] [blame]
Joerg Roedeldb3c33c2011-09-27 15:57:13 +02001/*
2 * drivers/pci/ats.c
3 *
4 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
Joerg Roedelc320b972011-09-27 15:57:15 +02005 * Copyright (C) 2011 Advanced Micro Devices,
Joerg Roedeldb3c33c2011-09-27 15:57:13 +02006 *
7 * PCI Express I/O Virtualization (IOV) support.
8 * Address Translation Service 1.0
Joerg Roedelc320b972011-09-27 15:57:15 +02009 * Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com>
Joerg Roedel086ac112011-09-27 15:57:16 +020010 * PASID support added by Joerg Roedel <joerg.roedel@amd.com>
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020011 */
12
Paul Gortmaker363c75d2011-05-27 09:37:25 -040013#include <linux/export.h>
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020014#include <linux/pci-ats.h>
15#include <linux/pci.h>
James Bottomley8c451942011-11-29 19:20:23 +000016#include <linux/slab.h>
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020017
18#include "pci.h"
19
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050020static void ats_alloc_one(struct pci_dev *dev)
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020021{
22 int pos;
23 u16 cap;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020024
25 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS);
26 if (!pos)
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050027 return;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020028
Bjorn Helgaasd544d752015-07-17 15:15:19 -050029 dev->ats_cap = pos;
30 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap);
31 dev->ats_qdep = PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) :
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020032 PCI_ATS_MAX_QDEP;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020033}
34
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050035void pci_ats_init(struct pci_dev *dev)
36{
37 ats_alloc_one(dev);
38}
39
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020040/**
41 * pci_enable_ats - enable the ATS capability
42 * @dev: the PCI device
43 * @ps: the IOMMU page shift
44 *
45 * Returns 0 on success, or negative on failure.
46 */
47int pci_enable_ats(struct pci_dev *dev, int ps)
48{
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020049 u16 ctrl;
50
Bjorn Helgaasd544d752015-07-17 15:15:19 -050051 BUG_ON(dev->ats_cap && dev->ats_enabled);
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020052
Bjorn Helgaasd544d752015-07-17 15:15:19 -050053 if (!dev->ats_cap)
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050054 return -EINVAL;
55
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020056 if (ps < PCI_ATS_MIN_STU)
57 return -EINVAL;
58
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050059 /*
60 * Note that enabling ATS on a VF fails unless it's already enabled
61 * with the same STU on the PF.
62 */
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020063 ctrl = PCI_ATS_CTRL_ENABLE;
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050064 if (dev->is_virtfn) {
65 struct pci_dev *pdev = dev->physfn;
66
Bjorn Helgaasd544d752015-07-17 15:15:19 -050067 if (pdev->ats_stu != ps)
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050068 return -EINVAL;
69
Bjorn Helgaasd544d752015-07-17 15:15:19 -050070 atomic_inc(&pdev->ats_ref_cnt); /* count enabled VFs */
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050071 } else {
Bjorn Helgaasd544d752015-07-17 15:15:19 -050072 dev->ats_stu = ps;
73 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050074 }
Bjorn Helgaasd544d752015-07-17 15:15:19 -050075 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020076
Bjorn Helgaasd544d752015-07-17 15:15:19 -050077 dev->ats_enabled = 1;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020078 return 0;
79}
Joerg Roedeld4c06362011-09-27 15:57:14 +020080EXPORT_SYMBOL_GPL(pci_enable_ats);
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020081
82/**
83 * pci_disable_ats - disable the ATS capability
84 * @dev: the PCI device
85 */
86void pci_disable_ats(struct pci_dev *dev)
87{
88 u16 ctrl;
89
Bjorn Helgaasd544d752015-07-17 15:15:19 -050090 BUG_ON(!dev->ats_cap || !dev->ats_enabled);
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020091
Bjorn Helgaasd544d752015-07-17 15:15:19 -050092 if (atomic_read(&dev->ats_ref_cnt))
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050093 return; /* VFs still enabled */
94
95 if (dev->is_virtfn) {
96 struct pci_dev *pdev = dev->physfn;
97
Bjorn Helgaasd544d752015-07-17 15:15:19 -050098 atomic_dec(&pdev->ats_ref_cnt);
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050099 }
100
Bjorn Helgaasd544d752015-07-17 15:15:19 -0500101 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl);
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200102 ctrl &= ~PCI_ATS_CTRL_ENABLE;
Bjorn Helgaasd544d752015-07-17 15:15:19 -0500103 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200104
Bjorn Helgaasd544d752015-07-17 15:15:19 -0500105 dev->ats_enabled = 0;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200106}
Joerg Roedeld4c06362011-09-27 15:57:14 +0200107EXPORT_SYMBOL_GPL(pci_disable_ats);
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200108
Hao, Xudong1900ca12011-12-17 21:24:40 +0800109void pci_restore_ats_state(struct pci_dev *dev)
110{
111 u16 ctrl;
112
113 if (!pci_ats_enabled(dev))
114 return;
115 if (!pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS))
116 BUG();
117
118 ctrl = PCI_ATS_CTRL_ENABLE;
119 if (!dev->is_virtfn)
Bjorn Helgaasd544d752015-07-17 15:15:19 -0500120 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
121 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
Hao, Xudong1900ca12011-12-17 21:24:40 +0800122}
123EXPORT_SYMBOL_GPL(pci_restore_ats_state);
124
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200125/**
126 * pci_ats_queue_depth - query the ATS Invalidate Queue Depth
127 * @dev: the PCI device
128 *
129 * Returns the queue depth on success, or negative on failure.
130 *
131 * The ATS spec uses 0 in the Invalidate Queue Depth field to
132 * indicate that the function can accept 32 Invalidate Request.
133 * But here we use the `real' values (i.e. 1~32) for the Queue
134 * Depth; and 0 indicates the function shares the Queue with
135 * other functions (doesn't exclusively own a Queue).
136 */
137int pci_ats_queue_depth(struct pci_dev *dev)
138{
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200139 if (dev->is_virtfn)
140 return 0;
141
Bjorn Helgaasd544d752015-07-17 15:15:19 -0500142 if (dev->ats_cap)
143 return dev->ats_qdep;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200144
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -0500145 return -ENODEV;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200146}
Joerg Roedeld4c06362011-09-27 15:57:14 +0200147EXPORT_SYMBOL_GPL(pci_ats_queue_depth);
Joerg Roedelc320b972011-09-27 15:57:15 +0200148
149#ifdef CONFIG_PCI_PRI
150/**
151 * pci_enable_pri - Enable PRI capability
152 * @ pdev: PCI device structure
153 *
154 * Returns 0 on success, negative value on error
155 */
156int pci_enable_pri(struct pci_dev *pdev, u32 reqs)
157{
158 u16 control, status;
159 u32 max_requests;
160 int pos;
161
Alex Williamson69166fb2011-11-02 14:07:15 -0600162 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
Joerg Roedelc320b972011-09-27 15:57:15 +0200163 if (!pos)
164 return -EINVAL;
165
Alex Williamson91f57d52011-11-11 10:07:36 -0700166 pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
167 pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status);
168 if ((control & PCI_PRI_CTRL_ENABLE) ||
169 !(status & PCI_PRI_STATUS_STOPPED))
Joerg Roedelc320b972011-09-27 15:57:15 +0200170 return -EBUSY;
171
Alex Williamson91f57d52011-11-11 10:07:36 -0700172 pci_read_config_dword(pdev, pos + PCI_PRI_MAX_REQ, &max_requests);
Joerg Roedelc320b972011-09-27 15:57:15 +0200173 reqs = min(max_requests, reqs);
Alex Williamson91f57d52011-11-11 10:07:36 -0700174 pci_write_config_dword(pdev, pos + PCI_PRI_ALLOC_REQ, reqs);
Joerg Roedelc320b972011-09-27 15:57:15 +0200175
Alex Williamson91f57d52011-11-11 10:07:36 -0700176 control |= PCI_PRI_CTRL_ENABLE;
177 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
Joerg Roedelc320b972011-09-27 15:57:15 +0200178
179 return 0;
180}
181EXPORT_SYMBOL_GPL(pci_enable_pri);
182
183/**
184 * pci_disable_pri - Disable PRI capability
185 * @pdev: PCI device structure
186 *
187 * Only clears the enabled-bit, regardless of its former value
188 */
189void pci_disable_pri(struct pci_dev *pdev)
190{
191 u16 control;
192 int pos;
193
Alex Williamson69166fb2011-11-02 14:07:15 -0600194 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
Joerg Roedelc320b972011-09-27 15:57:15 +0200195 if (!pos)
196 return;
197
Alex Williamson91f57d52011-11-11 10:07:36 -0700198 pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
199 control &= ~PCI_PRI_CTRL_ENABLE;
200 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
Joerg Roedelc320b972011-09-27 15:57:15 +0200201}
202EXPORT_SYMBOL_GPL(pci_disable_pri);
203
204/**
Joerg Roedelc320b972011-09-27 15:57:15 +0200205 * pci_reset_pri - Resets device's PRI state
206 * @pdev: PCI device structure
207 *
208 * The PRI capability must be disabled before this function is called.
209 * Returns 0 on success, negative value on error.
210 */
211int pci_reset_pri(struct pci_dev *pdev)
212{
213 u16 control;
214 int pos;
215
Alex Williamson69166fb2011-11-02 14:07:15 -0600216 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
Joerg Roedelc320b972011-09-27 15:57:15 +0200217 if (!pos)
218 return -EINVAL;
219
Alex Williamson91f57d52011-11-11 10:07:36 -0700220 pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
221 if (control & PCI_PRI_CTRL_ENABLE)
Joerg Roedelc320b972011-09-27 15:57:15 +0200222 return -EBUSY;
223
Alex Williamson91f57d52011-11-11 10:07:36 -0700224 control |= PCI_PRI_CTRL_RESET;
Joerg Roedelc320b972011-09-27 15:57:15 +0200225
Alex Williamson91f57d52011-11-11 10:07:36 -0700226 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
Joerg Roedelc320b972011-09-27 15:57:15 +0200227
228 return 0;
229}
230EXPORT_SYMBOL_GPL(pci_reset_pri);
Joerg Roedelc320b972011-09-27 15:57:15 +0200231#endif /* CONFIG_PCI_PRI */
Joerg Roedel086ac112011-09-27 15:57:16 +0200232
233#ifdef CONFIG_PCI_PASID
234/**
235 * pci_enable_pasid - Enable the PASID capability
236 * @pdev: PCI device structure
237 * @features: Features to enable
238 *
239 * Returns 0 on success, negative value on error. This function checks
240 * whether the features are actually supported by the device and returns
241 * an error if not.
242 */
243int pci_enable_pasid(struct pci_dev *pdev, int features)
244{
245 u16 control, supported;
246 int pos;
247
Alex Williamson69166fb2011-11-02 14:07:15 -0600248 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
Joerg Roedel086ac112011-09-27 15:57:16 +0200249 if (!pos)
250 return -EINVAL;
251
Alex Williamson91f57d52011-11-11 10:07:36 -0700252 pci_read_config_word(pdev, pos + PCI_PASID_CTRL, &control);
253 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported);
Joerg Roedel086ac112011-09-27 15:57:16 +0200254
Alex Williamson91f57d52011-11-11 10:07:36 -0700255 if (control & PCI_PASID_CTRL_ENABLE)
Joerg Roedel086ac112011-09-27 15:57:16 +0200256 return -EINVAL;
257
Alex Williamson91f57d52011-11-11 10:07:36 -0700258 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
Joerg Roedel086ac112011-09-27 15:57:16 +0200259
260 /* User wants to enable anything unsupported? */
261 if ((supported & features) != features)
262 return -EINVAL;
263
Alex Williamson91f57d52011-11-11 10:07:36 -0700264 control = PCI_PASID_CTRL_ENABLE | features;
Joerg Roedel086ac112011-09-27 15:57:16 +0200265
Alex Williamson91f57d52011-11-11 10:07:36 -0700266 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control);
Joerg Roedel086ac112011-09-27 15:57:16 +0200267
268 return 0;
269}
270EXPORT_SYMBOL_GPL(pci_enable_pasid);
271
272/**
273 * pci_disable_pasid - Disable the PASID capability
274 * @pdev: PCI device structure
275 *
276 */
277void pci_disable_pasid(struct pci_dev *pdev)
278{
279 u16 control = 0;
280 int pos;
281
Alex Williamson69166fb2011-11-02 14:07:15 -0600282 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
Joerg Roedel086ac112011-09-27 15:57:16 +0200283 if (!pos)
284 return;
285
Alex Williamson91f57d52011-11-11 10:07:36 -0700286 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control);
Joerg Roedel086ac112011-09-27 15:57:16 +0200287}
288EXPORT_SYMBOL_GPL(pci_disable_pasid);
289
290/**
291 * pci_pasid_features - Check which PASID features are supported
292 * @pdev: PCI device structure
293 *
294 * Returns a negative value when no PASI capability is present.
295 * Otherwise is returns a bitmask with supported features. Current
296 * features reported are:
Alex Williamson91f57d52011-11-11 10:07:36 -0700297 * PCI_PASID_CAP_EXEC - Execute permission supported
Bjorn Helgaasf7625982013-11-14 11:28:18 -0700298 * PCI_PASID_CAP_PRIV - Privileged mode supported
Joerg Roedel086ac112011-09-27 15:57:16 +0200299 */
300int pci_pasid_features(struct pci_dev *pdev)
301{
302 u16 supported;
303 int pos;
304
Alex Williamson69166fb2011-11-02 14:07:15 -0600305 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
Joerg Roedel086ac112011-09-27 15:57:16 +0200306 if (!pos)
307 return -EINVAL;
308
Alex Williamson91f57d52011-11-11 10:07:36 -0700309 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported);
Joerg Roedel086ac112011-09-27 15:57:16 +0200310
Alex Williamson91f57d52011-11-11 10:07:36 -0700311 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
Joerg Roedel086ac112011-09-27 15:57:16 +0200312
313 return supported;
314}
315EXPORT_SYMBOL_GPL(pci_pasid_features);
316
317#define PASID_NUMBER_SHIFT 8
318#define PASID_NUMBER_MASK (0x1f << PASID_NUMBER_SHIFT)
319/**
320 * pci_max_pasid - Get maximum number of PASIDs supported by device
321 * @pdev: PCI device structure
322 *
323 * Returns negative value when PASID capability is not present.
324 * Otherwise it returns the numer of supported PASIDs.
325 */
326int pci_max_pasids(struct pci_dev *pdev)
327{
328 u16 supported;
329 int pos;
330
Alex Williamson69166fb2011-11-02 14:07:15 -0600331 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
Joerg Roedel086ac112011-09-27 15:57:16 +0200332 if (!pos)
333 return -EINVAL;
334
Alex Williamson91f57d52011-11-11 10:07:36 -0700335 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported);
Joerg Roedel086ac112011-09-27 15:57:16 +0200336
337 supported = (supported & PASID_NUMBER_MASK) >> PASID_NUMBER_SHIFT;
338
339 return (1 << supported);
340}
341EXPORT_SYMBOL_GPL(pci_max_pasids);
342#endif /* CONFIG_PCI_PASID */