blob: 3c898c12a06b51c8926d85a3d3c5dda5d549a57f [file] [log] [blame]
Sarah Sharp66d4ead2009-04-27 19:52:28 -07001/*
2 * xHCI host controller driver PCI Bus Glue.
3 *
4 * Copyright (C) 2008 Intel Corp.
5 *
6 * Author: Sarah Sharp
7 * Some code borrowed from the Linux EHCI driver.
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
15 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 * for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software Foundation,
20 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 */
22
23#include <linux/pci.h>
Ben Hutchings7fc2a612011-04-25 16:54:28 +010024#include <linux/slab.h>
Paul Gortmaker6eb0de82011-07-03 16:09:31 -040025#include <linux/module.h>
Sarah Sharp66d4ead2009-04-27 19:52:28 -070026
27#include "xhci.h"
Xenia Ragiadakou4bdfe4c2013-08-06 07:52:45 +030028#include "xhci-trace.h"
Sarah Sharp66d4ead2009-04-27 19:52:28 -070029
Sarah Sharpac9d8fe2009-08-07 14:04:55 -070030/* Device for a quirk */
31#define PCI_VENDOR_ID_FRESCO_LOGIC 0x1b73
32#define PCI_DEVICE_ID_FRESCO_LOGIC_PDK 0x1000
Sarah Sharpbba18e32012-10-17 13:44:06 -070033#define PCI_DEVICE_ID_FRESCO_LOGIC_FL1400 0x1400
Sarah Sharpac9d8fe2009-08-07 14:04:55 -070034
Maarten Lankhorstc877b3b2011-06-15 23:47:21 +020035#define PCI_VENDOR_ID_ETRON 0x1b6f
36#define PCI_DEVICE_ID_ASROCK_P67 0x7023
37
Takashi Iwai638298d2013-09-12 08:11:06 +020038#define PCI_DEVICE_ID_INTEL_LYNXPOINT_XHCI 0x8c31
39#define PCI_DEVICE_ID_INTEL_LYNXPOINT_LP_XHCI 0x9c31
40
Sarah Sharp66d4ead2009-04-27 19:52:28 -070041static const char hcd_name[] = "xhci_hcd";
42
43/* called after powerup, by probe or system-pm "wakeup" */
44static int xhci_pci_reinit(struct xhci_hcd *xhci, struct pci_dev *pdev)
45{
46 /*
47 * TODO: Implement finding debug ports later.
48 * TODO: see if there are any quirks that need to be added to handle
49 * new extended capabilities.
50 */
51
52 /* PCI Memory-Write-Invalidate cycle support is optional (uncommon) */
53 if (!pci_set_mwi(pdev))
54 xhci_dbg(xhci, "MWI active\n");
55
56 xhci_dbg(xhci, "Finished xhci_pci_reinit\n");
57 return 0;
58}
59
Sebastian Andrzej Siewiorda3c9c42011-09-23 14:20:00 -070060static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci)
61{
62 struct pci_dev *pdev = to_pci_dev(dev);
63
Sarah Sharpac9d8fe2009-08-07 14:04:55 -070064 /* Look for vendor-specific quirks */
65 if (pdev->vendor == PCI_VENDOR_ID_FRESCO_LOGIC &&
Sarah Sharpbba18e32012-10-17 13:44:06 -070066 (pdev->device == PCI_DEVICE_ID_FRESCO_LOGIC_PDK ||
67 pdev->device == PCI_DEVICE_ID_FRESCO_LOGIC_FL1400)) {
68 if (pdev->device == PCI_DEVICE_ID_FRESCO_LOGIC_PDK &&
69 pdev->revision == 0x0) {
Sarah Sharpac9d8fe2009-08-07 14:04:55 -070070 xhci->quirks |= XHCI_RESET_EP_QUIRK;
Xenia Ragiadakou4bdfe4c2013-08-06 07:52:45 +030071 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
72 "QUIRK: Fresco Logic xHC needs configure"
73 " endpoint cmd after reset endpoint");
Sarah Sharpf5182b42011-06-02 11:33:02 -070074 }
Oliver Neukum455f5892013-09-30 15:50:54 +020075 if (pdev->device == PCI_DEVICE_ID_FRESCO_LOGIC_PDK &&
76 pdev->revision == 0x4) {
77 xhci->quirks |= XHCI_SLOW_SUSPEND;
78 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
79 "QUIRK: Fresco Logic xHC revision %u"
80 "must be suspended extra slowly",
81 pdev->revision);
82 }
Sarah Sharpf5182b42011-06-02 11:33:02 -070083 /* Fresco Logic confirms: all revisions of this chip do not
84 * support MSI, even though some of them claim to in their PCI
85 * capabilities.
86 */
87 xhci->quirks |= XHCI_BROKEN_MSI;
Xenia Ragiadakou4bdfe4c2013-08-06 07:52:45 +030088 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
89 "QUIRK: Fresco Logic revision %u "
90 "has broken MSI implementation",
Sarah Sharpf5182b42011-06-02 11:33:02 -070091 pdev->revision);
Sarah Sharp1530bbc62012-05-08 09:22:49 -070092 xhci->quirks |= XHCI_TRUST_TX_LENGTH;
Sarah Sharpac9d8fe2009-08-07 14:04:55 -070093 }
Sarah Sharpf5182b42011-06-02 11:33:02 -070094
Sarah Sharp02386342010-05-24 13:25:28 -070095 if (pdev->vendor == PCI_VENDOR_ID_NEC)
96 xhci->quirks |= XHCI_NEC_HOST;
Sarah Sharpac9d8fe2009-08-07 14:04:55 -070097
Andiry Xu7e393a82011-09-23 14:19:54 -070098 if (pdev->vendor == PCI_VENDOR_ID_AMD && xhci->hci_version == 0x96)
99 xhci->quirks |= XHCI_AMD_0x96_HOST;
100
Andiry Xuc41136b2011-03-22 17:08:14 +0800101 /* AMD PLL quirk */
102 if (pdev->vendor == PCI_VENDOR_ID_AMD && usb_amd_find_chipset_info())
103 xhci->quirks |= XHCI_AMD_PLL_FIX;
Sarah Sharpe3567d22012-05-16 13:36:24 -0700104 if (pdev->vendor == PCI_VENDOR_ID_INTEL) {
105 xhci->quirks |= XHCI_LPM_SUPPORT;
106 xhci->quirks |= XHCI_INTEL_HOST;
107 }
Sarah Sharpad808332011-05-25 10:43:56 -0700108 if (pdev->vendor == PCI_VENDOR_ID_INTEL &&
109 pdev->device == PCI_DEVICE_ID_INTEL_PANTHERPOINT_XHCI) {
Sarah Sharp2cf95c12011-05-11 16:14:58 -0700110 xhci->quirks |= XHCI_EP_LIMIT_QUIRK;
111 xhci->limit_active_eps = 64;
Sarah Sharp86cc5582011-09-02 11:05:54 -0700112 xhci->quirks |= XHCI_SW_BW_CHECKING;
Sarah Sharpe95829f2012-07-23 18:59:30 +0300113 /*
114 * PPT desktop boards DH77EB and DH77DF will power back on after
115 * a few seconds of being shutdown. The fix for this is to
116 * switch the ports from xHCI to EHCI on shutdown. We can't use
117 * DMI information to find those particular boards (since each
118 * vendor will change the board name), so we have to key off all
119 * PPT chipsets.
120 */
121 xhci->quirks |= XHCI_SPURIOUS_REBOOT;
Sarah Sharp80fab3b2012-09-19 16:27:26 -0700122 xhci->quirks |= XHCI_AVOID_BEI;
Sarah Sharpad808332011-05-25 10:43:56 -0700123 }
Takashi Iwai638298d2013-09-12 08:11:06 +0200124 if (pdev->vendor == PCI_VENDOR_ID_INTEL &&
125 (pdev->device == PCI_DEVICE_ID_INTEL_LYNXPOINT_XHCI ||
126 pdev->device == PCI_DEVICE_ID_INTEL_LYNXPOINT_LP_XHCI)) {
127 /* Workaround for occasional spurious wakeups from S5 (or
128 * any other sleep) on Haswell machines with LPT and LPT-LP
129 * with the new Intel BIOS
130 */
Takashi Iwai6962d912013-12-09 14:53:36 +0100131 /* Limit the quirk to only known vendors, as this triggers
132 * yet another BIOS bug on some other machines
133 * https://bugzilla.kernel.org/show_bug.cgi?id=66171
134 */
135 if (pdev->subsystem_vendor == PCI_VENDOR_ID_HP)
136 xhci->quirks |= XHCI_SPURIOUS_WAKEUP;
Takashi Iwai638298d2013-09-12 08:11:06 +0200137 }
Maarten Lankhorstc877b3b2011-06-15 23:47:21 +0200138 if (pdev->vendor == PCI_VENDOR_ID_ETRON &&
139 pdev->device == PCI_DEVICE_ID_ASROCK_P67) {
140 xhci->quirks |= XHCI_RESET_ON_RESUME;
Xenia Ragiadakou4bdfe4c2013-08-06 07:52:45 +0300141 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
142 "QUIRK: Resetting on resume");
Sarah Sharp5cb7df22012-07-02 13:36:23 -0700143 xhci->quirks |= XHCI_TRUST_TX_LENGTH;
Maarten Lankhorstc877b3b2011-06-15 23:47:21 +0200144 }
Elric Fu457a4f62012-03-29 15:47:50 +0800145 if (pdev->vendor == PCI_VENDOR_ID_VIA)
146 xhci->quirks |= XHCI_RESET_ON_RESUME;
Sebastian Andrzej Siewiorda3c9c42011-09-23 14:20:00 -0700147}
Andiry Xuc41136b2011-03-22 17:08:14 +0800148
Sebastian Andrzej Siewiorda3c9c42011-09-23 14:20:00 -0700149/* called during probe() after chip reset completes */
150static int xhci_pci_setup(struct usb_hcd *hcd)
151{
152 struct xhci_hcd *xhci;
153 struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
154 int retval;
155
156 retval = xhci_gen_setup(hcd, xhci_pci_quirks);
Sarah Sharp66d4ead2009-04-27 19:52:28 -0700157 if (retval)
Sebastian Andrzej Siewiorda3c9c42011-09-23 14:20:00 -0700158 return retval;
Sarah Sharp66d4ead2009-04-27 19:52:28 -0700159
Sebastian Andrzej Siewiorda3c9c42011-09-23 14:20:00 -0700160 xhci = hcd_to_xhci(hcd);
161 if (!usb_hcd_is_primary_hcd(hcd))
162 return 0;
Sarah Sharp66d4ead2009-04-27 19:52:28 -0700163
164 pci_read_config_byte(pdev, XHCI_SBRN_OFFSET, &xhci->sbrn);
165 xhci_dbg(xhci, "Got SBRN %u\n", (unsigned int) xhci->sbrn);
166
167 /* Find any debug ports */
Sarah Sharpb02d0ed2010-10-26 11:03:44 -0700168 retval = xhci_pci_reinit(xhci, pdev);
169 if (!retval)
170 return retval;
171
Sarah Sharpb02d0ed2010-10-26 11:03:44 -0700172 kfree(xhci);
173 return retval;
174}
175
Sarah Sharpf6ff0ac2010-12-16 11:21:10 -0800176/*
177 * We need to register our own PCI probe function (instead of the USB core's
178 * function) in order to create a second roothub under xHCI.
179 */
180static int xhci_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
181{
182 int retval;
183 struct xhci_hcd *xhci;
184 struct hc_driver *driver;
185 struct usb_hcd *hcd;
186
187 driver = (struct hc_driver *)id->driver_data;
188 /* Register the USB 2.0 roothub.
189 * FIXME: USB core must know to register the USB 2.0 roothub first.
190 * This is sort of silly, because we could just set the HCD driver flags
191 * to say USB 2.0, but I'm not sure what the implications would be in
192 * the other parts of the HCD code.
193 */
194 retval = usb_hcd_pci_probe(dev, id);
195
196 if (retval)
197 return retval;
198
199 /* USB 2.0 roothub is stored in the PCI device now. */
200 hcd = dev_get_drvdata(&dev->dev);
201 xhci = hcd_to_xhci(hcd);
202 xhci->shared_hcd = usb_create_shared_hcd(driver, &dev->dev,
203 pci_name(dev), hcd);
204 if (!xhci->shared_hcd) {
205 retval = -ENOMEM;
206 goto dealloc_usb2_hcd;
207 }
208
209 /* Set the xHCI pointer before xhci_pci_setup() (aka hcd_driver.reset)
210 * is called by usb_add_hcd().
211 */
212 *((struct xhci_hcd **) xhci->shared_hcd->hcd_priv) = xhci;
213
214 retval = usb_add_hcd(xhci->shared_hcd, dev->irq,
Yong Zhangb5dd18d2011-09-07 16:10:52 +0800215 IRQF_SHARED);
Sarah Sharpf6ff0ac2010-12-16 11:21:10 -0800216 if (retval)
217 goto put_usb3_hcd;
218 /* Roothub already marked as USB 3.0 speed */
Sarah Sharp3b3db022012-05-09 10:55:03 -0700219
220 /* We know the LPM timeout algorithms for this host, let the USB core
221 * enable and disable LPM for devices under the USB 3.0 roothub.
222 */
223 if (xhci->quirks & XHCI_LPM_SUPPORT)
224 hcd_to_bus(xhci->shared_hcd)->root_hub->lpm_capable = 1;
225
Sarah Sharpf6ff0ac2010-12-16 11:21:10 -0800226 return 0;
227
228put_usb3_hcd:
229 usb_put_hcd(xhci->shared_hcd);
230dealloc_usb2_hcd:
231 usb_hcd_pci_remove(dev);
232 return retval;
233}
234
Sarah Sharpb02d0ed2010-10-26 11:03:44 -0700235static void xhci_pci_remove(struct pci_dev *dev)
236{
237 struct xhci_hcd *xhci;
238
239 xhci = hcd_to_xhci(pci_get_drvdata(dev));
Sarah Sharpf6ff0ac2010-12-16 11:21:10 -0800240 if (xhci->shared_hcd) {
241 usb_remove_hcd(xhci->shared_hcd);
242 usb_put_hcd(xhci->shared_hcd);
243 }
Sarah Sharpb02d0ed2010-10-26 11:03:44 -0700244 usb_hcd_pci_remove(dev);
Takashi Iwai638298d2013-09-12 08:11:06 +0200245
246 /* Workaround for spurious wakeups at shutdown with HSW */
247 if (xhci->quirks & XHCI_SPURIOUS_WAKEUP)
248 pci_set_power_state(dev, PCI_D3hot);
249
Sarah Sharpb02d0ed2010-10-26 11:03:44 -0700250 kfree(xhci);
Sarah Sharp66d4ead2009-04-27 19:52:28 -0700251}
252
Andiry Xu5535b1d2010-10-14 07:23:06 -0700253#ifdef CONFIG_PM
254static int xhci_pci_suspend(struct usb_hcd *hcd, bool do_wakeup)
255{
256 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
Sarah Sharpc3897aa2013-04-18 10:02:03 -0700257 struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
258
259 /*
260 * Systems with the TI redriver that loses port status change events
261 * need to have the registers polled during D3, so avoid D3cold.
262 */
263 if (xhci_compliance_mode_recovery_timer_quirk_check())
264 pdev->no_d3cold = true;
Andiry Xu5535b1d2010-10-14 07:23:06 -0700265
Felipe Balbi77b84762012-10-19 10:55:16 +0300266 return xhci_suspend(xhci);
Andiry Xu5535b1d2010-10-14 07:23:06 -0700267}
268
269static int xhci_pci_resume(struct usb_hcd *hcd, bool hibernated)
270{
271 struct xhci_hcd *xhci = hcd_to_xhci(hcd);
Sarah Sharp69e848c2011-02-22 09:57:15 -0800272 struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
Andiry Xu5535b1d2010-10-14 07:23:06 -0700273 int retval = 0;
274
Sarah Sharp69e848c2011-02-22 09:57:15 -0800275 /* The BIOS on systems with the Intel Panther Point chipset may or may
276 * not support xHCI natively. That means that during system resume, it
277 * may switch the ports back to EHCI so that users can use their
278 * keyboard to select a kernel from GRUB after resume from hibernate.
279 *
280 * The BIOS is supposed to remember whether the OS had xHCI ports
281 * enabled before resume, and switch the ports back to xHCI when the
282 * BIOS/OS semaphore is written, but we all know we can't trust BIOS
283 * writers.
284 *
285 * Unconditionally switch the ports back to xHCI after a system resume.
Mathias Nyman26b76792013-07-23 11:35:47 +0300286 * It should not matter whether the EHCI or xHCI controller is
287 * resumed first. It's enough to do the switchover in xHCI because
288 * USB core won't notice anything as the hub driver doesn't start
289 * running again until after all the devices (including both EHCI and
290 * xHCI host controllers) have been resumed.
Sarah Sharp69e848c2011-02-22 09:57:15 -0800291 */
Mathias Nyman26b76792013-07-23 11:35:47 +0300292
293 if (pdev->vendor == PCI_VENDOR_ID_INTEL)
294 usb_enable_intel_xhci_ports(pdev);
Sarah Sharp69e848c2011-02-22 09:57:15 -0800295
Andiry Xu5535b1d2010-10-14 07:23:06 -0700296 retval = xhci_resume(xhci, hibernated);
297 return retval;
298}
299#endif /* CONFIG_PM */
300
Sarah Sharp66d4ead2009-04-27 19:52:28 -0700301static const struct hc_driver xhci_pci_hc_driver = {
302 .description = hcd_name,
303 .product_desc = "xHCI Host Controller",
Sarah Sharpb02d0ed2010-10-26 11:03:44 -0700304 .hcd_priv_size = sizeof(struct xhci_hcd *),
Sarah Sharp66d4ead2009-04-27 19:52:28 -0700305
306 /*
307 * generic hardware linkage
308 */
Sarah Sharp7f84eef2009-04-27 19:53:56 -0700309 .irq = xhci_irq,
Sarah Sharpf6ff0ac2010-12-16 11:21:10 -0800310 .flags = HCD_MEMORY | HCD_USB3 | HCD_SHARED,
Sarah Sharp66d4ead2009-04-27 19:52:28 -0700311
312 /*
313 * basic lifecycle operations
314 */
315 .reset = xhci_pci_setup,
316 .start = xhci_run,
Andiry Xu5535b1d2010-10-14 07:23:06 -0700317#ifdef CONFIG_PM
318 .pci_suspend = xhci_pci_suspend,
319 .pci_resume = xhci_pci_resume,
320#endif
Sarah Sharp66d4ead2009-04-27 19:52:28 -0700321 .stop = xhci_stop,
322 .shutdown = xhci_shutdown,
323
324 /*
Sarah Sharp3ffbba92009-04-27 19:57:38 -0700325 * managing i/o requests and associated device resources
326 */
Sarah Sharpd0e96f5a2009-04-27 19:58:01 -0700327 .urb_enqueue = xhci_urb_enqueue,
328 .urb_dequeue = xhci_urb_dequeue,
Sarah Sharp3ffbba92009-04-27 19:57:38 -0700329 .alloc_dev = xhci_alloc_dev,
330 .free_dev = xhci_free_dev,
Sarah Sharpeab1caf2010-04-05 10:55:58 -0700331 .alloc_streams = xhci_alloc_streams,
332 .free_streams = xhci_free_streams,
Sarah Sharpf94e01862009-04-27 19:58:38 -0700333 .add_endpoint = xhci_add_endpoint,
334 .drop_endpoint = xhci_drop_endpoint,
Sarah Sharpa1587d92009-07-27 12:03:15 -0700335 .endpoint_reset = xhci_endpoint_reset,
Sarah Sharpf94e01862009-04-27 19:58:38 -0700336 .check_bandwidth = xhci_check_bandwidth,
337 .reset_bandwidth = xhci_reset_bandwidth,
Sarah Sharp3ffbba92009-04-27 19:57:38 -0700338 .address_device = xhci_address_device,
Dan Williams48fc7db2013-12-05 17:07:27 -0800339 .enable_device = xhci_enable_device,
Sarah Sharpb356b7c2009-09-04 10:53:24 -0700340 .update_hub_device = xhci_update_hub_device,
Andiry Xuf0615c42010-10-14 07:22:48 -0700341 .reset_device = xhci_discover_or_reset_device,
Sarah Sharp3ffbba92009-04-27 19:57:38 -0700342
343 /*
Sarah Sharp66d4ead2009-04-27 19:52:28 -0700344 * scheduling support
345 */
346 .get_frame_number = xhci_get_frame,
347
Sarah Sharp0f2a7932009-04-27 19:57:12 -0700348 /* Root hub support */
349 .hub_control = xhci_hub_control,
350 .hub_status_data = xhci_hub_status_data,
Andiry Xu9777e3c2010-10-14 07:23:03 -0700351 .bus_suspend = xhci_bus_suspend,
352 .bus_resume = xhci_bus_resume,
Andiry Xu95743232011-09-23 14:19:51 -0700353 /*
354 * call back when device connected and addressed
355 */
356 .update_device = xhci_update_device,
Andiry Xu65580b432011-09-23 14:19:52 -0700357 .set_usb2_hw_lpm = xhci_set_usb2_hardware_lpm,
Sarah Sharp3b3db022012-05-09 10:55:03 -0700358 .enable_usb3_lpm_timeout = xhci_enable_usb3_lpm_timeout,
359 .disable_usb3_lpm_timeout = xhci_disable_usb3_lpm_timeout,
Lan Tianyu3f5eb142013-03-19 16:48:12 +0800360 .find_raw_port_number = xhci_find_raw_port_number,
Sarah Sharp66d4ead2009-04-27 19:52:28 -0700361};
362
363/*-------------------------------------------------------------------------*/
364
365/* PCI driver selection metadata; PCI hotplugging uses this */
366static const struct pci_device_id pci_ids[] = { {
367 /* handle any USB 3.0 xHCI controller */
368 PCI_DEVICE_CLASS(PCI_CLASS_SERIAL_USB_XHCI, ~0),
369 .driver_data = (unsigned long) &xhci_pci_hc_driver,
370 },
371 { /* end: all zeroes */ }
372};
373MODULE_DEVICE_TABLE(pci, pci_ids);
374
375/* pci driver glue; this is a "new style" PCI driver module */
376static struct pci_driver xhci_pci_driver = {
377 .name = (char *) hcd_name,
378 .id_table = pci_ids,
379
Sarah Sharpf6ff0ac2010-12-16 11:21:10 -0800380 .probe = xhci_pci_probe,
Sarah Sharpb02d0ed2010-10-26 11:03:44 -0700381 .remove = xhci_pci_remove,
Sarah Sharp66d4ead2009-04-27 19:52:28 -0700382 /* suspend and resume implemented later */
383
384 .shutdown = usb_hcd_pci_shutdown,
Alan Sternf875fdb2013-09-24 15:45:25 -0400385#ifdef CONFIG_PM
Andiry Xu5535b1d2010-10-14 07:23:06 -0700386 .driver = {
387 .pm = &usb_hcd_pci_pm_ops
388 },
389#endif
Sarah Sharp66d4ead2009-04-27 19:52:28 -0700390};
391
Sebastian Andrzej Siewior0cc47d52011-09-23 14:20:02 -0700392int __init xhci_register_pci(void)
Sarah Sharp66d4ead2009-04-27 19:52:28 -0700393{
394 return pci_register_driver(&xhci_pci_driver);
395}
396
Gerard Snitselaara46c46a2012-03-16 11:34:11 -0700397void xhci_unregister_pci(void)
Sarah Sharp66d4ead2009-04-27 19:52:28 -0700398{
399 pci_unregister_driver(&xhci_pci_driver);
400}