drivers/pci/host/pci-msm.c

/* Copyright (c) 2014-2018, The Linux Foundation. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 and
 * only version 2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

/*
 * MSM PCIe controller driver.
 */

#include <linux/module.h>
#include <linux/bitops.h>
#include <linux/clk.h>
#include <linux/debugfs.h>
#include <linux/delay.h>
#include <linux/jiffies.h>
#include <linux/gpio.h>
#include <linux/iopoll.h>
#include <linux/kernel.h>
#include <linux/of_pci.h>
#include <linux/pci.h>
#include <linux/iommu.h>
#include <linux/platform_device.h>
#include <linux/regulator/consumer.h>
#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/of_gpio.h>
#include <linux/clk/qcom.h>
#include <linux/reset.h>
#include <linux/msm-bus.h>
#include <linux/msm-bus-board.h>
#include <linux/seq_file.h>
#include <linux/debugfs.h>
#include <linux/uaccess.h>
#include <linux/io.h>
#include <linux/msi.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/irqdomain.h>
#include <linux/pm_wakeup.h>
#include <linux/compiler.h>
#include <soc/qcom/scm.h>
#include <linux/ipc_logging.h>
#include <linux/msm_pcie.h>

#define PCIE_VENDOR_ID_QCOM		0x17cb

#define PCIE20_L1SUB_CONTROL1		0x1E4
#define PCIE20_PARF_DBI_BASE_ADDR       0x350
#define PCIE20_PARF_SLV_ADDR_SPACE_SIZE 0x358

#define PCS_BASE 0x800

#define PCS_PORT(n) (PCS_BASE + n * 0x1000)

#define PCIE_N_SW_RESET(n)			(PCS_PORT(n) + 0x00)
#define PCIE_N_POWER_DOWN_CONTROL(n)		(PCS_PORT(n) + 0x04)

#define PCIE_GEN3_COM_INTEGLOOP_GAIN1_MODE0	0x0154
#define PCIE_GEN3_L0_DRVR_CTRL0			0x080c
#define PCIE_GEN3_L0_RESET_GEN			0x0890
#define PCIE_GEN3_L0_BIST_ERR_CNT1_STATUS	0x08a8
#define PCIE_GEN3_L0_BIST_ERR_CNT2_STATUS	0x08ac
#define PCIE_GEN3_L0_DEBUG_BUS_STATUS4		0x08bc

#define PCIE20_PARF_SYS_CTRL	     0x00
#define PCIE20_PARF_PM_CTRL		0x20
#define PCIE20_PARF_PM_STTS		0x24
#define PCIE20_PARF_PCS_DEEMPH	   0x34
#define PCIE20_PARF_PCS_SWING	    0x38
#define PCIE20_PARF_PHY_CTRL	     0x40
#define PCIE20_PARF_PHY_REFCLK	   0x4C
#define PCIE20_PARF_CONFIG_BITS	  0x50
#define PCIE20_PARF_TEST_BUS		0xE4
#define PCIE20_PARF_MHI_CLOCK_RESET_CTRL	0x174
#define PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT   0x1A8
#define PCIE20_PARF_LTSSM              0x1B0
#define PCIE20_PARF_INT_ALL_STATUS	0x224
#define PCIE20_PARF_INT_ALL_CLEAR	0x228
#define PCIE20_PARF_INT_ALL_MASK	0x22C
#define PCIE20_PARF_SID_OFFSET		0x234
#define PCIE20_PARF_BDF_TRANSLATE_CFG	0x24C
#define PCIE20_PARF_BDF_TRANSLATE_N	0x250
#define PCIE20_PARF_DEVICE_TYPE		0x1000

#define PCIE20_ELBI_VERSION		0x00
#define PCIE20_ELBI_SYS_CTRL	     0x04
#define PCIE20_ELBI_SYS_STTS		 0x08

#define PCIE20_CAP			   0x70
#define PCIE20_CAP_DEVCTRLSTATUS	(PCIE20_CAP + 0x08)
#define PCIE20_CAP_LINKCTRLSTATUS	(PCIE20_CAP + 0x10)

#define PCIE20_COMMAND_STATUS	    0x04
#define PCIE20_HEADER_TYPE		0x0C
#define PCIE20_BUSNUMBERS		  0x18
#define PCIE20_MEMORY_BASE_LIMIT	 0x20
#define PCIE20_BRIDGE_CTRL		0x3C
#define PCIE20_DEVICE_CONTROL_STATUS	0x78
#define PCIE20_DEVICE_CONTROL2_STATUS2 0x98

#define PCIE20_AUX_CLK_FREQ_REG		0xB40
#define PCIE20_ACK_F_ASPM_CTRL_REG     0x70C
#define PCIE20_ACK_N_FTS		   0xff00

#define PCIE20_PLR_IATU_VIEWPORT	 0x900
#define PCIE20_PLR_IATU_CTRL1	    0x904
#define PCIE20_PLR_IATU_CTRL2	    0x908
#define PCIE20_PLR_IATU_LBAR	     0x90C
#define PCIE20_PLR_IATU_UBAR	     0x910
#define PCIE20_PLR_IATU_LAR		0x914
#define PCIE20_PLR_IATU_LTAR	     0x918
#define PCIE20_PLR_IATU_UTAR	     0x91c

#define PCIE_IATU_BASE(n)	(n * 0x200)

#define PCIE_IATU_CTRL1(n)	(PCIE_IATU_BASE(n) + 0x00)
#define PCIE_IATU_CTRL2(n)	(PCIE_IATU_BASE(n) + 0x04)
#define PCIE_IATU_LBAR(n)	(PCIE_IATU_BASE(n) + 0x08)
#define PCIE_IATU_UBAR(n)	(PCIE_IATU_BASE(n) + 0x0c)
#define PCIE_IATU_LAR(n)	(PCIE_IATU_BASE(n) + 0x10)
#define PCIE_IATU_LTAR(n)	(PCIE_IATU_BASE(n) + 0x14)
#define PCIE_IATU_UTAR(n)	(PCIE_IATU_BASE(n) + 0x18)

#define PCIE20_PORT_LINK_CTRL_REG	0x710
#define PCIE20_GEN3_RELATED_REG	0x890
#define PCIE20_PIPE_LOOPBACK_CONTROL	0x8b8
#define LOOPBACK_BASE_ADDR_OFFSET	0x8000

#define PCIE20_CTRL1_TYPE_CFG0		0x04
#define PCIE20_CTRL1_TYPE_CFG1		0x05

#define PCIE20_CAP_ID			0x10
#define L1SUB_CAP_ID			0x1E

#define PCIE_CAP_PTR_OFFSET		0x34
#define PCIE_EXT_CAP_OFFSET		0x100

#define PCIE20_AER_UNCORR_ERR_STATUS_REG	0x104
#define PCIE20_AER_CORR_ERR_STATUS_REG		0x110
#define PCIE20_AER_ROOT_ERR_STATUS_REG		0x130
#define PCIE20_AER_ERR_SRC_ID_REG		0x134

#define RD 0
#define WR 1
#define MSM_PCIE_ERROR -1

#define PERST_PROPAGATION_DELAY_US_MIN	  1000
#define PERST_PROPAGATION_DELAY_US_MAX	  1005
#define SWITCH_DELAY_MAX	  20
#define REFCLK_STABILIZATION_DELAY_US_MIN     1000
#define REFCLK_STABILIZATION_DELAY_US_MAX     1005
#define LINK_UP_TIMEOUT_US_MIN		    5000
#define LINK_UP_TIMEOUT_US_MAX		    5100
#define LINK_UP_CHECK_MAX_COUNT		   20
#define EP_UP_TIMEOUT_US_MIN	1000
#define EP_UP_TIMEOUT_US_MAX	1005
#define EP_UP_TIMEOUT_US	1000000
#define PHY_STABILIZATION_DELAY_US_MIN	  995
#define PHY_STABILIZATION_DELAY_US_MAX	  1005
#define POWER_DOWN_DELAY_US_MIN		10
#define POWER_DOWN_DELAY_US_MAX		11
#define LINKDOWN_INIT_WAITING_US_MIN    995
#define LINKDOWN_INIT_WAITING_US_MAX    1005
#define LINKDOWN_WAITING_US_MIN	   4900
#define LINKDOWN_WAITING_US_MAX	   5100
#define LINKDOWN_WAITING_COUNT	    200

#define GEN1_SPEED 0x1
#define GEN2_SPEED 0x2
#define GEN3_SPEED 0x3

#define PHY_READY_TIMEOUT_COUNT		   10
#define XMLH_LINK_UP				  0x400
#define MAX_LINK_RETRIES 5
#define MAX_BUS_NUM 3
#define MAX_PROP_SIZE 32
#define MAX_RC_NAME_LEN 15
#define MSM_PCIE_MAX_VREG 4
#define MSM_PCIE_MAX_CLK 13
#define MSM_PCIE_MAX_PIPE_CLK 1
#define MAX_RC_NUM 3
#define MAX_DEVICE_NUM 20
#define MAX_SHORT_BDF_NUM 16
#define PCIE_TLP_RD_SIZE 0x5
#define PCIE_MSI_NR_IRQS 256
#define MSM_PCIE_MAX_MSI 32
#define PCIE_LOG_PAGES (50)
#define PCIE_CONF_SPACE_DW			1024
#define PCIE_CLEAR				0xDEADBEEF
#define PCIE_LINK_DOWN				0xFFFFFFFF

#define MSM_PCIE_MAX_RESET 5
#define MSM_PCIE_MAX_PIPE_RESET 1

#define MSM_PCIE_MSI_PHY 0xa0000000
#define PCIE20_MSI_CTRL_ADDR		(0x820)
#define PCIE20_MSI_CTRL_UPPER_ADDR	(0x824)
#define PCIE20_MSI_CTRL_INTR_EN	   (0x828)
#define PCIE20_MSI_CTRL_INTR_MASK	 (0x82C)
#define PCIE20_MSI_CTRL_INTR_STATUS     (0x830)
#define PCIE20_MSI_CTRL_MAX 8

/* PM control options */
#define PM_IRQ			 0x1
#define PM_CLK			 0x2
#define PM_GPIO			0x4
#define PM_VREG			0x8
#define PM_PIPE_CLK		  0x10
#define PM_ALL (PM_IRQ | PM_CLK | PM_GPIO | PM_VREG | PM_PIPE_CLK)

#ifdef CONFIG_PHYS_ADDR_T_64BIT
#define PCIE_UPPER_ADDR(addr) ((u32)((addr) >> 32))
#else
#define PCIE_UPPER_ADDR(addr) (0x0)
#endif
#define PCIE_LOWER_ADDR(addr) ((u32)((addr) & 0xffffffff))

#define PCIE_BUS_PRIV_DATA(bus) \
	(struct msm_pcie_dev_t *)(bus->sysdata)

/* Config Space Offsets */
#define BDF_OFFSET(bus, devfn) \
	((bus << 24) | (devfn << 16))

#define PCIE_GEN_DBG(x...) do { \
	if (msm_pcie_debug_mask) \
		pr_alert(x); \
	} while (0)

#define PCIE_DBG(dev, fmt, arg...) do {			 \
	if ((dev) && (dev)->ipc_log_long)   \
		ipc_log_string((dev)->ipc_log_long, \
			"DBG1:%s: " fmt, __func__, arg); \
	if ((dev) && (dev)->ipc_log)   \
		ipc_log_string((dev)->ipc_log, "%s: " fmt, __func__, arg); \
	if (msm_pcie_debug_mask)   \
		pr_alert("%s: " fmt, __func__, arg);		  \
	} while (0)

#define PCIE_DBG2(dev, fmt, arg...) do {			 \
	if ((dev) && (dev)->ipc_log)   \
		ipc_log_string((dev)->ipc_log, "DBG2:%s: " fmt, __func__, arg);\
	if (msm_pcie_debug_mask)   \
		pr_alert("%s: " fmt, __func__, arg);              \
	} while (0)

#define PCIE_DBG3(dev, fmt, arg...) do {			 \
	if ((dev) && (dev)->ipc_log)   \
		ipc_log_string((dev)->ipc_log, "DBG3:%s: " fmt, __func__, arg);\
	if (msm_pcie_debug_mask)   \
		pr_alert("%s: " fmt, __func__, arg);              \
	} while (0)

#define PCIE_DUMP(dev, fmt, arg...) do {			\
	if ((dev) && (dev)->ipc_log_dump) \
		ipc_log_string((dev)->ipc_log_dump, \
			"DUMP:%s: " fmt, __func__, arg); \
	} while (0)

#define PCIE_DBG_FS(dev, fmt, arg...) do {			\
	if ((dev) && (dev)->ipc_log_dump) \
		ipc_log_string((dev)->ipc_log_dump, \
			"DBG_FS:%s: " fmt, __func__, arg); \
	pr_alert("%s: " fmt, __func__, arg); \
	} while (0)

#define PCIE_INFO(dev, fmt, arg...) do {			 \
	if ((dev) && (dev)->ipc_log_long)   \
		ipc_log_string((dev)->ipc_log_long, \
			"INFO:%s: " fmt, __func__, arg); \
	if ((dev) && (dev)->ipc_log)   \
		ipc_log_string((dev)->ipc_log, "%s: " fmt, __func__, arg); \
	pr_info("%s: " fmt, __func__, arg);  \
	} while (0)

#define PCIE_ERR(dev, fmt, arg...) do {			 \
	if ((dev) && (dev)->ipc_log_long)   \
		ipc_log_string((dev)->ipc_log_long, \
			"ERR:%s: " fmt, __func__, arg); \
	if ((dev) && (dev)->ipc_log)   \
		ipc_log_string((dev)->ipc_log, "%s: " fmt, __func__, arg); \
	pr_err("%s: " fmt, __func__, arg);  \
	} while (0)


enum msm_pcie_res {
	MSM_PCIE_RES_PARF,
	MSM_PCIE_RES_PHY,
	MSM_PCIE_RES_DM_CORE,
	MSM_PCIE_RES_ELBI,
	MSM_PCIE_RES_IATU,
	MSM_PCIE_RES_CONF,
	MSM_PCIE_RES_IO,
	MSM_PCIE_RES_BARS,
	MSM_PCIE_RES_TCSR,
	MSM_PCIE_MAX_RES,
};

enum msm_pcie_irq {
	MSM_PCIE_INT_MSI,
	MSM_PCIE_INT_A,
	MSM_PCIE_INT_B,
	MSM_PCIE_INT_C,
	MSM_PCIE_INT_D,
	MSM_PCIE_INT_PLS_PME,
	MSM_PCIE_INT_PME_LEGACY,
	MSM_PCIE_INT_PLS_ERR,
	MSM_PCIE_INT_AER_LEGACY,
	MSM_PCIE_INT_LINK_UP,
	MSM_PCIE_INT_LINK_DOWN,
	MSM_PCIE_INT_BRIDGE_FLUSH_N,
	MSM_PCIE_INT_GLOBAL_INT,
	MSM_PCIE_MAX_IRQ,
};

enum msm_pcie_irq_event {
	MSM_PCIE_INT_EVT_LINK_DOWN = 1,
	MSM_PCIE_INT_EVT_BME,
	MSM_PCIE_INT_EVT_PM_TURNOFF,
	MSM_PCIE_INT_EVT_DEBUG,
	MSM_PCIE_INT_EVT_LTR,
	MSM_PCIE_INT_EVT_MHI_Q6,
	MSM_PCIE_INT_EVT_MHI_A7,
	MSM_PCIE_INT_EVT_DSTATE_CHANGE,
	MSM_PCIE_INT_EVT_L1SUB_TIMEOUT,
	MSM_PCIE_INT_EVT_MMIO_WRITE,
	MSM_PCIE_INT_EVT_CFG_WRITE,
	MSM_PCIE_INT_EVT_BRIDGE_FLUSH_N,
	MSM_PCIE_INT_EVT_LINK_UP,
	MSM_PCIE_INT_EVT_AER_LEGACY,
	MSM_PCIE_INT_EVT_AER_ERR,
	MSM_PCIE_INT_EVT_PME_LEGACY,
	MSM_PCIE_INT_EVT_PLS_PME,
	MSM_PCIE_INT_EVT_INTD,
	MSM_PCIE_INT_EVT_INTC,
	MSM_PCIE_INT_EVT_INTB,
	MSM_PCIE_INT_EVT_INTA,
	MSM_PCIE_INT_EVT_EDMA,
	MSM_PCIE_INT_EVT_MSI_0,
	MSM_PCIE_INT_EVT_MSI_1,
	MSM_PCIE_INT_EVT_MSI_2,
	MSM_PCIE_INT_EVT_MSI_3,
	MSM_PCIE_INT_EVT_MSI_4,
	MSM_PCIE_INT_EVT_MSI_5,
	MSM_PCIE_INT_EVT_MSI_6,
	MSM_PCIE_INT_EVT_MSI_7,
	MSM_PCIE_INT_EVT_MAX = 30,
};

enum msm_pcie_gpio {
	MSM_PCIE_GPIO_PERST,
	MSM_PCIE_GPIO_WAKE,
	MSM_PCIE_GPIO_EP,
	MSM_PCIE_MAX_GPIO
};

enum msm_pcie_link_status {
	MSM_PCIE_LINK_DEINIT,
	MSM_PCIE_LINK_ENABLED,
	MSM_PCIE_LINK_DISABLED
};

enum msm_pcie_boot_option {
	MSM_PCIE_NO_PROBE_ENUMERATION = BIT(0),
	MSM_PCIE_NO_WAKE_ENUMERATION = BIT(1)
};

enum msm_pcie_debugfs_option {
	MSM_PCIE_OUTPUT_PCIE_INFO,
	MSM_PCIE_DISABLE_LINK,
	MSM_PCIE_ENABLE_LINK,
	MSM_PCIE_DISABLE_ENABLE_LINK,
	MSM_PCIE_DUMP_SHADOW_REGISTER,
	MSM_PCIE_DISABLE_L0S,
	MSM_PCIE_ENABLE_L0S,
	MSM_PCIE_DISABLE_L1,
	MSM_PCIE_ENABLE_L1,
	MSM_PCIE_DISABLE_L1SS,
	MSM_PCIE_ENABLE_L1SS,
	MSM_PCIE_ENUMERATION,
	MSM_PCIE_READ_PCIE_REGISTER,
	MSM_PCIE_WRITE_PCIE_REGISTER,
	MSM_PCIE_DUMP_PCIE_REGISTER_SPACE,
	MSM_PCIE_ALLOCATE_DDR_MAP_LBAR,
	MSM_PCIE_FREE_DDR_UNMAP_LBAR,
	MSM_PCIE_OUTPUT_DDR_LBAR_ADDRESS,
	MSM_PCIE_CONFIGURE_LOOPBACK,
	MSM_PCIE_SETUP_LOOPBACK_IATU,
	MSM_PCIE_READ_DDR,
	MSM_PCIE_READ_LBAR,
	MSM_PCIE_WRITE_DDR,
	MSM_PCIE_WRITE_LBAR,
	MSM_PCIE_DISABLE_AER,
	MSM_PCIE_ENABLE_AER,
	MSM_PCIE_GPIO_STATUS,
	MSM_PCIE_ASSERT_PERST,
	MSM_PCIE_DEASSERT_PERST,
	MSM_PCIE_KEEP_RESOURCES_ON,
	MSM_PCIE_FORCE_GEN1,
	MSM_PCIE_MAX_DEBUGFS_OPTION
};

static const char * const
	msm_pcie_debugfs_option_desc[MSM_PCIE_MAX_DEBUGFS_OPTION] = {
	"OUTPUT PCIE INFO",
	"DISABLE LINK",
	"ENABLE LINK",
	"DISABLE AND ENABLE LINK",
	"DUMP PCIE SHADOW REGISTER",
	"DISABLE L0S",
	"ENABLE L0S",
	"DISABLE L1",
	"ENABLE L1",
	"DISABLE L1SS",
	"ENABLE L1SS",
	"ENUMERATE",
	"READ A PCIE REGISTER",
	"WRITE TO PCIE REGISTER",
	"DUMP PCIE REGISTER SPACE",
	"ALLOCATE DDR AND MAP LBAR",
	"FREE DDR AND UNMAP LBAR",
	"OUTPUT DDR AND LBAR VIR ADDRESS",
	"CONFIGURE PCIE LOOPBACK",
	"SETUP LOOPBACK IATU",
	"READ DDR",
	"READ LBAR",
	"WRITE DDR",
	"WRITE LBAR",
	"SET AER ENABLE FLAG",
	"CLEAR AER ENABLE FLAG",
	"OUTPUT PERST AND WAKE GPIO STATUS",
	"ASSERT PERST",
	"DE-ASSERT PERST",
	"SET KEEP_RESOURCES_ON FLAG",
	"FORCE GEN 1 SPEED FOR LINK TRAINING"
};

/* gpio info structure */
struct msm_pcie_gpio_info_t {
	char	*name;
	uint32_t   num;
	bool	 out;
	uint32_t   on;
	uint32_t   init;
	bool	required;
};

/* voltage regulator info structrue */
struct msm_pcie_vreg_info_t {
	struct regulator  *hdl;
	char		  *name;
	uint32_t	     max_v;
	uint32_t	     min_v;
	uint32_t	     opt_mode;
	bool		   required;
};

/* reset info structure */
struct msm_pcie_reset_info_t {
	struct reset_control *hdl;
	char *name;
	bool required;
};

/* clock info structure */
struct msm_pcie_clk_info_t {
	struct clk  *hdl;
	char	  *name;
	u32	   freq;
	bool	config_mem;
	bool	  required;
};

/* resource info structure */
struct msm_pcie_res_info_t {
	char		*name;
	struct resource *resource;
	void __iomem    *base;
};

/* irq info structrue */
struct msm_pcie_irq_info_t {
	char		  *name;
	uint32_t	    num;
};

/* phy info structure */
struct msm_pcie_phy_info_t {
	u32	offset;
	u32	val;
	u32	delay;
};

/* PCIe device info structure */
struct msm_pcie_device_info {
	u32			bdf;
	struct pci_dev		*dev;
	short			short_bdf;
	u32			sid;
	int			domain;
	void __iomem		*conf_base;
	unsigned long		phy_address;
	u32			dev_ctrlstts_offset;
	struct msm_pcie_register_event *event_reg;
	bool			registered;
};

/* msm pcie device structure */
struct msm_pcie_dev_t {
	struct platform_device	 *pdev;
	struct pci_dev *dev;
	struct regulator *gdsc;
	struct regulator *gdsc_smmu;
	struct msm_pcie_vreg_info_t  vreg[MSM_PCIE_MAX_VREG];
	struct msm_pcie_gpio_info_t  gpio[MSM_PCIE_MAX_GPIO];
	struct msm_pcie_clk_info_t   clk[MSM_PCIE_MAX_CLK];
	struct msm_pcie_clk_info_t   pipeclk[MSM_PCIE_MAX_PIPE_CLK];
	struct msm_pcie_res_info_t   res[MSM_PCIE_MAX_RES];
	struct msm_pcie_irq_info_t   irq[MSM_PCIE_MAX_IRQ];
	struct msm_pcie_irq_info_t	msi[MSM_PCIE_MAX_MSI];
	struct msm_pcie_reset_info_t reset[MSM_PCIE_MAX_RESET];
	struct msm_pcie_reset_info_t pipe_reset[MSM_PCIE_MAX_PIPE_RESET];

	void __iomem		     *parf;
	void __iomem		     *phy;
	void __iomem		     *elbi;
	void __iomem		     *iatu;
	void __iomem		     *dm_core;
	void __iomem		     *conf;
	void __iomem		     *bars;
	void __iomem		     *tcsr;

	uint32_t			    axi_bar_start;
	uint32_t			    axi_bar_end;

	struct resource		   *dev_mem_res;
	struct resource		   *dev_io_res;

	uint32_t			    wake_n;
	uint32_t			    vreg_n;
	uint32_t			    gpio_n;
	uint32_t			    parf_deemph;
	uint32_t			    parf_swing;

	bool				 cfg_access;
	spinlock_t			 cfg_lock;
	unsigned long		    irqsave_flags;
	struct mutex			enumerate_lock;
	struct mutex		     setup_lock;

	struct irq_domain		*irq_domain;
	DECLARE_BITMAP(msi_irq_in_use, PCIE_MSI_NR_IRQS);
	uint32_t			   msi_gicm_addr;
	uint32_t			   msi_gicm_base;
	bool				 use_msi;

	enum msm_pcie_link_status    link_status;
	bool				 user_suspend;
	bool                         disable_pc;
	struct pci_saved_state	     *saved_state;

	struct wakeup_source	     ws;
	struct msm_bus_scale_pdata   *bus_scale_table;
	uint32_t			   bus_client;

	bool				l0s_supported;
	bool				l1_supported;
	bool				 l1ss_supported;
	bool				common_clk_en;
	bool				clk_power_manage_en;
	bool				 aux_clk_sync;
	bool				aer_enable;
	bool				smmu_exist;
	uint32_t			smmu_sid_base;
	uint32_t			   n_fts;
	uint32_t			max_link_speed;
	bool				 ext_ref_clk;
	uint32_t			   ep_latency;
	uint32_t			switch_latency;
	uint32_t			wr_halt_size;
	uint32_t			slv_addr_space_size;
	uint32_t			phy_status_offset;
	uint32_t			cpl_timeout;
	uint32_t			current_bdf;
	uint32_t			perst_delay_us_min;
	uint32_t			perst_delay_us_max;
	uint32_t			tlp_rd_size;
	bool				linkdown_panic;
	uint32_t			boot_option;

	uint32_t			   rc_idx;
	uint32_t			phy_ver;
	bool				drv_ready;
	bool				 enumerated;
	struct work_struct	     handle_wake_work;
	struct mutex		     recovery_lock;
	spinlock_t                   wakeup_lock;
	spinlock_t			irq_lock;
	ulong				linkdown_counter;
	ulong				link_turned_on_counter;
	ulong				link_turned_off_counter;
	ulong				rc_corr_counter;
	ulong				rc_non_fatal_counter;
	ulong				rc_fatal_counter;
	ulong				ep_corr_counter;
	ulong				ep_non_fatal_counter;
	ulong				ep_fatal_counter;
	bool				 suspending;
	ulong				wake_counter;
	u32				num_active_ep;
	u32				num_ep;
	bool				pending_ep_reg;
	u32				phy_len;
	struct msm_pcie_phy_info_t	*phy_sequence;
	u32		ep_shadow[MAX_DEVICE_NUM][PCIE_CONF_SPACE_DW];
	u32				  rc_shadow[PCIE_CONF_SPACE_DW];
	bool				 shadow_en;
	bool				bridge_found;
	struct msm_pcie_register_event *event_reg;
	unsigned int			scm_dev_id;
	bool				 power_on;
	void				 *ipc_log;
	void				*ipc_log_long;
	void				*ipc_log_dump;
	bool				use_19p2mhz_aux_clk;
	bool				use_pinctrl;
	struct pinctrl			*pinctrl;
	struct pinctrl_state		*pins_default;
	struct pinctrl_state		*pins_sleep;
	struct msm_pcie_device_info   pcidev_table[MAX_DEVICE_NUM];
};

/* debug mask sys interface */
static int msm_pcie_debug_mask;
module_param_named(debug_mask, msm_pcie_debug_mask,
			    int, 0644);

/*
 * For each bit set, invert the default capability
 * option for the corresponding root complex
 * and its devices.
 */
static int msm_pcie_invert_l0s_support;
module_param_named(invert_l0s_support, msm_pcie_invert_l0s_support,
			    int, 0644);
static int msm_pcie_invert_l1_support;
module_param_named(invert_l1_support, msm_pcie_invert_l1_support,
			    int, 0644);
static int msm_pcie_invert_l1ss_support;
module_param_named(invert_l1ss_support, msm_pcie_invert_l1ss_support,
			    int, 0644);
static int msm_pcie_invert_aer_support;
module_param_named(invert_aer_support, msm_pcie_invert_aer_support,
			    int, 0644);

/*
 * For each bit set, keep the resources on when link training fails
 * or linkdown occurs for the corresponding root complex
 */
static int msm_pcie_keep_resources_on;
module_param_named(keep_resources_on, msm_pcie_keep_resources_on,
			    int, 0644);

/*
 * For each bit set, force the corresponding root complex
 * to do link training at gen1 speed.
 */
static int msm_pcie_force_gen1;
module_param_named(force_gen1, msm_pcie_force_gen1,
			    int, 0644);


/*
 * For each bit set in BIT[3:0] determines which corresponding
 * root complex will use the value in BIT[31:4] to override the
 * default (LINK_UP_CHECK_MAX_COUNT) max check count for link training.
 * Each iteration is LINK_UP_TIMEOUT_US_MIN long.
 */
static int msm_pcie_link_check_max_count;
module_param_named(link_check_max_count, msm_pcie_link_check_max_count,
			    int, 0644);

/* debugfs values */
static u32 rc_sel = BIT(0);
static u32 base_sel;
static u32 wr_offset;
static u32 wr_mask;
static u32 wr_value;
static u32 corr_counter_limit = 5;

/* Table to track info of PCIe devices */
static struct msm_pcie_device_info
	msm_pcie_dev_tbl[MAX_RC_NUM * MAX_DEVICE_NUM];

/* PCIe driver state */
static struct pcie_drv_sta {
	u32 rc_num;
	struct mutex drv_lock;
} pcie_drv;

/* msm pcie device data */
static struct msm_pcie_dev_t msm_pcie_dev[MAX_RC_NUM];

/* regulators */
static struct msm_pcie_vreg_info_t msm_pcie_vreg_info[MSM_PCIE_MAX_VREG] = {
	{NULL, "vreg-3.3", 0, 0, 0, false},
	{NULL, "vreg-1.8", 1800000, 1800000, 14000, true},
	{NULL, "vreg-0.9", 1000000, 1000000, 40000, true},
	{NULL, "vreg-cx", 0, 0, 0, false}
};

/* GPIOs */
static struct msm_pcie_gpio_info_t msm_pcie_gpio_info[MSM_PCIE_MAX_GPIO] = {
	{"perst-gpio",		0, 1, 0, 0, 1},
	{"wake-gpio",		0, 0, 0, 0, 0},
	{"qcom,ep-gpio",	0, 1, 1, 0, 0}
};

/* resets */
static struct msm_pcie_reset_info_t
msm_pcie_reset_info[MAX_RC_NUM][MSM_PCIE_MAX_RESET] = {
	{
		{NULL, "pcie_0_core_reset", false},
		{NULL, "pcie_phy_reset", false},
		{NULL, "pcie_phy_com_reset", false},
		{NULL, "pcie_phy_nocsr_com_phy_reset", false},
		{NULL, "pcie_0_phy_reset", false}
	},
	{
		{NULL, "pcie_1_core_reset", false},
		{NULL, "pcie_phy_reset", false},
		{NULL, "pcie_phy_com_reset", false},
		{NULL, "pcie_phy_nocsr_com_phy_reset", false},
		{NULL, "pcie_1_phy_reset", false}
	},
	{
		{NULL, "pcie_2_core_reset", false},
		{NULL, "pcie_phy_reset", false},
		{NULL, "pcie_phy_com_reset", false},
		{NULL, "pcie_phy_nocsr_com_phy_reset", false},
		{NULL, "pcie_2_phy_reset", false}
	}
};

/* pipe reset  */
static struct msm_pcie_reset_info_t
msm_pcie_pipe_reset_info[MAX_RC_NUM][MSM_PCIE_MAX_PIPE_RESET] = {
	{
		{NULL, "pcie_0_phy_pipe_reset", false}
	},
	{
		{NULL, "pcie_1_phy_pipe_reset", false}
	},
	{
		{NULL, "pcie_2_phy_pipe_reset", false}
	}
};

/* clocks */
static struct msm_pcie_clk_info_t
	msm_pcie_clk_info[MAX_RC_NUM][MSM_PCIE_MAX_CLK] = {
	{
	{NULL, "pcie_0_ref_clk_src", 0, false, false},
	{NULL, "pcie_0_aux_clk", 1010000, false, true},
	{NULL, "pcie_0_cfg_ahb_clk", 0, false, true},
	{NULL, "pcie_0_mstr_axi_clk", 0, true, true},
	{NULL, "pcie_0_slv_axi_clk", 0, true, true},
	{NULL, "pcie_0_ldo", 0, false, true},
	{NULL, "pcie_0_smmu_clk", 0, false, false},
	{NULL, "pcie_0_slv_q2a_axi_clk", 0, false, false},
	{NULL, "pcie_0_sleep_clk", 0, false, false},
	{NULL, "pcie_phy_refgen_clk", 0, false, false},
	{NULL, "pcie_tbu_clk", 0, false, false},
	{NULL, "pcie_phy_cfg_ahb_clk", 0, false, false},
	{NULL, "pcie_phy_aux_clk", 0, false, false}
	},
	{
	{NULL, "pcie_1_ref_clk_src", 0, false, false},
	{NULL, "pcie_1_aux_clk", 1010000, false, true},
	{NULL, "pcie_1_cfg_ahb_clk", 0, false, true},
	{NULL, "pcie_1_mstr_axi_clk", 0, true, true},
	{NULL, "pcie_1_slv_axi_clk", 0, true,  true},
	{NULL, "pcie_1_ldo", 0, false, true},
	{NULL, "pcie_1_smmu_clk", 0, false, false},
	{NULL, "pcie_1_slv_q2a_axi_clk", 0, false, false},
	{NULL, "pcie_1_sleep_clk", 0, false, false},
	{NULL, "pcie_phy_refgen_clk", 0, false, false},
	{NULL, "pcie_tbu_clk", 0, false, false},
	{NULL, "pcie_phy_cfg_ahb_clk", 0, false, false},
	{NULL, "pcie_phy_aux_clk", 0, false, false}
	},
	{
	{NULL, "pcie_2_ref_clk_src", 0, false, false},
	{NULL, "pcie_2_aux_clk", 1010000, false, true},
	{NULL, "pcie_2_cfg_ahb_clk", 0, false, true},
	{NULL, "pcie_2_mstr_axi_clk", 0, true, true},
	{NULL, "pcie_2_slv_axi_clk", 0, true, true},
	{NULL, "pcie_2_ldo", 0, false, true},
	{NULL, "pcie_2_smmu_clk", 0, false, false},
	{NULL, "pcie_2_slv_q2a_axi_clk", 0, false, false},
	{NULL, "pcie_2_sleep_clk", 0, false, false},
	{NULL, "pcie_phy_refgen_clk", 0, false, false},
	{NULL, "pcie_tbu_clk", 0, false, false},
	{NULL, "pcie_phy_cfg_ahb_clk", 0, false, false},
	{NULL, "pcie_phy_aux_clk", 0, false, false}
	}
};

/* Pipe Clocks */
static struct msm_pcie_clk_info_t
	msm_pcie_pipe_clk_info[MAX_RC_NUM][MSM_PCIE_MAX_PIPE_CLK] = {
	{
	{NULL, "pcie_0_pipe_clk", 125000000, true, true},
	},
	{
	{NULL, "pcie_1_pipe_clk", 125000000, true, true},
	},
	{
	{NULL, "pcie_2_pipe_clk", 125000000, true, true},
	}
};

/* resources */
static const struct msm_pcie_res_info_t msm_pcie_res_info[MSM_PCIE_MAX_RES] = {
	{"parf",	NULL, NULL},
	{"phy",     NULL, NULL},
	{"dm_core",	NULL, NULL},
	{"elbi",	NULL, NULL},
	{"iatu",	NULL, NULL},
	{"conf",	NULL, NULL},
	{"io",		NULL, NULL},
	{"bars",	NULL, NULL},
	{"tcsr",	NULL, NULL}
};

/* irqs */
static const struct msm_pcie_irq_info_t msm_pcie_irq_info[MSM_PCIE_MAX_IRQ] = {
	{"int_msi",	0},
	{"int_a",	0},
	{"int_b",	0},
	{"int_c",	0},
	{"int_d",	0},
	{"int_pls_pme",		0},
	{"int_pme_legacy",	0},
	{"int_pls_err",		0},
	{"int_aer_legacy",	0},
	{"int_pls_link_up",	0},
	{"int_pls_link_down",	0},
	{"int_bridge_flush_n",	0},
	{"int_global_int",	0}
};

/* MSIs */
static const struct msm_pcie_irq_info_t msm_pcie_msi_info[MSM_PCIE_MAX_MSI] = {
	{"msi_0", 0}, {"msi_1", 0}, {"msi_2", 0}, {"msi_3", 0},
	{"msi_4", 0}, {"msi_5", 0}, {"msi_6", 0}, {"msi_7", 0},
	{"msi_8", 0}, {"msi_9", 0}, {"msi_10", 0}, {"msi_11", 0},
	{"msi_12", 0}, {"msi_13", 0}, {"msi_14", 0}, {"msi_15", 0},
	{"msi_16", 0}, {"msi_17", 0}, {"msi_18", 0}, {"msi_19", 0},
	{"msi_20", 0}, {"msi_21", 0}, {"msi_22", 0}, {"msi_23", 0},
	{"msi_24", 0}, {"msi_25", 0}, {"msi_26", 0}, {"msi_27", 0},
	{"msi_28", 0}, {"msi_29", 0}, {"msi_30", 0}, {"msi_31", 0}
};

static int msm_pcie_config_device(struct pci_dev *dev, void *pdev);
static int msm_pcie_config_l0s_disable(struct pci_dev *dev, void *pdev);
static int msm_pcie_config_l0s_enable(struct pci_dev *dev, void *pdev);
static int msm_pcie_config_l1_disable(struct pci_dev *dev, void *pdev);
static int msm_pcie_config_l1_enable(struct pci_dev *dev, void *pdev);
static int msm_pcie_config_l1ss_disable(struct pci_dev *dev, void *pdev);
static int msm_pcie_config_l1ss_enable(struct pci_dev *dev, void *pdev);
static void msm_pcie_config_link_pm_rc(struct msm_pcie_dev_t *dev,
				struct pci_dev *pdev, bool enable);

#ifdef CONFIG_ARM
static inline void msm_pcie_fixup_irqs(struct msm_pcie_dev_t *dev)
{
	pci_fixup_irqs(pci_common_swizzle, of_irq_parse_and_map_pci);
}
#else
static inline void msm_pcie_fixup_irqs(struct msm_pcie_dev_t *dev)
{
}
#endif

static inline void msm_pcie_write_reg(void __iomem *base, u32 offset, u32 value)
{
	writel_relaxed(value, base + offset);
	/* ensure that changes propagated to the hardware */
	wmb();
}

static inline void msm_pcie_write_reg_field(void __iomem *base, u32 offset,
	const u32 mask, u32 val)
{
	u32 shift = find_first_bit((void *)&mask, 32);
	u32 tmp = readl_relaxed(base + offset);

	tmp &= ~mask; /* clear written bits */
	val = tmp | (val << shift);
	writel_relaxed(val, base + offset);
	/* ensure that changes propagated to the hardware */
	wmb();
}

static inline void msm_pcie_config_clear_set_dword(struct pci_dev *pdev,
	int pos, u32 clear, u32 set)
{
	u32 val;

	pci_read_config_dword(pdev, pos, &val);
	val &= ~clear;
	val |= set;
	pci_write_config_dword(pdev, pos, val);
}

static inline void msm_pcie_config_clock_mem(struct msm_pcie_dev_t *dev,
	struct msm_pcie_clk_info_t *info)
{
	int ret;

	ret = clk_set_flags(info->hdl, CLKFLAG_NORETAIN_MEM);
	if (ret)
		PCIE_ERR(dev,
			"PCIe: RC%d can't configure core memory for clk %s: %d.\n",
			dev->rc_idx, info->name, ret);
	else
		PCIE_DBG2(dev,
			"PCIe: RC%d configured core memory for clk %s.\n",
			dev->rc_idx, info->name);

	ret = clk_set_flags(info->hdl, CLKFLAG_NORETAIN_PERIPH);
	if (ret)
		PCIE_ERR(dev,
			"PCIe: RC%d can't configure peripheral memory for clk %s: %d.\n",
			dev->rc_idx, info->name, ret);
	else
		PCIE_DBG2(dev,
			"PCIe: RC%d configured peripheral memory for clk %s.\n",
			dev->rc_idx, info->name);
}

static void pcie_phy_dump(struct msm_pcie_dev_t *dev)
{
	int i, size;

	size = resource_size(dev->res[MSM_PCIE_RES_PHY].resource);
	for (i = 0; i < size; i += 32) {
		PCIE_DUMP(dev,
			"PCIe PHY of RC%d: 0x%04x %08x %08x %08x %08x %08x %08x %08x %08x\n",
			dev->rc_idx, i,
			readl_relaxed(dev->phy + i),
			readl_relaxed(dev->phy + (i + 4)),
			readl_relaxed(dev->phy + (i + 8)),
			readl_relaxed(dev->phy + (i + 12)),
			readl_relaxed(dev->phy + (i + 16)),
			readl_relaxed(dev->phy + (i + 20)),
			readl_relaxed(dev->phy + (i + 24)),
			readl_relaxed(dev->phy + (i + 28)));
	}
}

static void pcie_phy_init(struct msm_pcie_dev_t *dev)
{
	int i;
	struct msm_pcie_phy_info_t *phy_seq;

	PCIE_DBG(dev,
		"RC%d: Initializing 14nm QMP phy - 19.2MHz with Common Mode Clock (SSC ON)\n",
		dev->rc_idx);

	if (dev->phy_sequence) {
		i =  dev->phy_len;
		phy_seq = dev->phy_sequence;
		while (i--) {
			msm_pcie_write_reg(dev->phy,
				phy_seq->offset,
				phy_seq->val);
			if (phy_seq->delay)
				usleep_range(phy_seq->delay,
					phy_seq->delay + 1);
			phy_seq++;
		}
	}
}

static bool pcie_phy_is_ready(struct msm_pcie_dev_t *dev)
{
	if (readl_relaxed(dev->phy + dev->phy_status_offset) & BIT(6))
		return false;
	else
		return true;
}

static int msm_pcie_restore_sec_config(struct msm_pcie_dev_t *dev)
{
	int ret, scm_ret;

	if (!dev) {
		pr_err("PCIe: the input pcie dev is NULL.\n");
		return -ENODEV;
	}

	ret = scm_restore_sec_cfg(dev->scm_dev_id, 0, &scm_ret);
	if (ret || scm_ret) {
		PCIE_ERR(dev,
			"PCIe: RC%d failed(%d) to restore sec config, scm_ret=%d\n",
			dev->rc_idx, ret, scm_ret);
		return ret ? ret : -EINVAL;
	}

	return 0;
}

static inline int msm_pcie_check_align(struct msm_pcie_dev_t *dev,
						u32 offset)
{
	if (offset % 4) {
		PCIE_ERR(dev,
			"PCIe: RC%d: offset 0x%x is not correctly aligned\n",
			dev->rc_idx, offset);
		return MSM_PCIE_ERROR;
	}

	return 0;
}

static bool msm_pcie_confirm_linkup(struct msm_pcie_dev_t *dev,
						bool check_sw_stts,
						bool check_ep,
						void __iomem *ep_conf)
{
	u32 val;

	if (check_sw_stts && (dev->link_status != MSM_PCIE_LINK_ENABLED)) {
		PCIE_DBG(dev, "PCIe: The link of RC %d is not enabled.\n",
			dev->rc_idx);
		return false;
	}

	if (!(readl_relaxed(dev->dm_core + 0x80) & BIT(29))) {
		PCIE_DBG(dev, "PCIe: The link of RC %d is not up.\n",
			dev->rc_idx);
		return false;
	}

	val = readl_relaxed(dev->dm_core);
	PCIE_DBG(dev, "PCIe: device ID and vender ID of RC %d are 0x%x.\n",
		dev->rc_idx, val);
	if (val == PCIE_LINK_DOWN) {
		PCIE_ERR(dev,
			"PCIe: The link of RC %d is not really up; device ID and vender ID of RC %d are 0x%x.\n",
			dev->rc_idx, dev->rc_idx, val);
		return false;
	}

	if (check_ep) {
		val = readl_relaxed(ep_conf);
		PCIE_DBG(dev,
			"PCIe: device ID and vender ID of EP of RC %d are 0x%x.\n",
			dev->rc_idx, val);
		if (val == PCIE_LINK_DOWN) {
			PCIE_ERR(dev,
				"PCIe: The link of RC %d is not really up; device ID and vender ID of EP of RC %d are 0x%x.\n",
				dev->rc_idx, dev->rc_idx, val);
			return false;
		}
	}

	return true;
}

static void msm_pcie_cfg_recover(struct msm_pcie_dev_t *dev, bool rc)
{
	int i, j;
	u32 val = 0;
	u32 *shadow;
	void __iomem *cfg = dev->conf;

	for (i = 0; i < MAX_DEVICE_NUM; i++) {
		if (!rc && !dev->pcidev_table[i].bdf)
			break;
		if (rc) {
			cfg = dev->dm_core;
			shadow = dev->rc_shadow;
		} else {
			if (!msm_pcie_confirm_linkup(dev, false, true,
				dev->pcidev_table[i].conf_base))
				continue;

			shadow = dev->ep_shadow[i];
			PCIE_DBG(dev,
				"PCIe Device: %02x:%02x.%01x\n",
				dev->pcidev_table[i].bdf >> 24,
				dev->pcidev_table[i].bdf >> 19 & 0x1f,
				dev->pcidev_table[i].bdf >> 16 & 0x07);
		}
		for (j = PCIE_CONF_SPACE_DW - 1; j >= 0; j--) {
			val = shadow[j];
			if (val != PCIE_CLEAR) {
				PCIE_DBG3(dev,
					"PCIe: before recovery:cfg 0x%x:0x%x\n",
					j * 4, readl_relaxed(cfg + j * 4));
				PCIE_DBG3(dev,
					"PCIe: shadow_dw[%d]:cfg 0x%x:0x%x\n",
					j, j * 4, val);
				writel_relaxed(val, cfg + j * 4);
				/* ensure changes propagated to the hardware */
				wmb();
				PCIE_DBG3(dev,
					"PCIe: after recovery:cfg 0x%x:0x%x\n\n",
					j * 4, readl_relaxed(cfg + j * 4));
			}
		}
		if (rc)
			break;

		pci_save_state(dev->pcidev_table[i].dev);
		cfg += SZ_4K;
	}
}

static void msm_pcie_write_mask(void __iomem *addr,
				uint32_t clear_mask, uint32_t set_mask)
{
	uint32_t val;

	val = (readl_relaxed(addr) & ~clear_mask) | set_mask;
	writel_relaxed(val, addr);
	wmb();  /* ensure data is written to hardware register */
}

static void pcie_parf_dump(struct msm_pcie_dev_t *dev)
{
	int i, size;
	u32 original;

	PCIE_DUMP(dev, "PCIe: RC%d PARF testbus\n", dev->rc_idx);

	original = readl_relaxed(dev->parf + PCIE20_PARF_SYS_CTRL);
	for (i = 1; i <= 0x1A; i++) {
		msm_pcie_write_mask(dev->parf + PCIE20_PARF_SYS_CTRL,
				0xFF0000, i << 16);
		PCIE_DUMP(dev,
			"RC%d: PARF_SYS_CTRL: 0%08x PARF_TEST_BUS: 0%08x\n",
			dev->rc_idx,
			readl_relaxed(dev->parf + PCIE20_PARF_SYS_CTRL),
			readl_relaxed(dev->parf + PCIE20_PARF_TEST_BUS));
	}
	writel_relaxed(original, dev->parf + PCIE20_PARF_SYS_CTRL);

	PCIE_DUMP(dev, "PCIe: RC%d PARF register dump\n", dev->rc_idx);

	size = resource_size(dev->res[MSM_PCIE_RES_PARF].resource);
	for (i = 0; i < size; i += 32) {
		PCIE_DUMP(dev,
			"RC%d: 0x%04x %08x %08x %08x %08x %08x %08x %08x %08x\n",
			dev->rc_idx, i,
			readl_relaxed(dev->parf + i),
			readl_relaxed(dev->parf + (i + 4)),
			readl_relaxed(dev->parf + (i + 8)),
			readl_relaxed(dev->parf + (i + 12)),
			readl_relaxed(dev->parf + (i + 16)),
			readl_relaxed(dev->parf + (i + 20)),
			readl_relaxed(dev->parf + (i + 24)),
			readl_relaxed(dev->parf + (i + 28)));
	}
}

static void msm_pcie_show_status(struct msm_pcie_dev_t *dev)
{
	PCIE_DBG_FS(dev, "PCIe: RC%d is %s enumerated\n",
		dev->rc_idx, dev->enumerated ? "" : "not");
	PCIE_DBG_FS(dev, "PCIe: link is %s\n",
		(dev->link_status == MSM_PCIE_LINK_ENABLED)
		? "enabled" : "disabled");
	PCIE_DBG_FS(dev, "cfg_access is %s allowed\n",
		dev->cfg_access ? "" : "not");
	PCIE_DBG_FS(dev, "use_msi is %d\n",
		dev->use_msi);
	PCIE_DBG_FS(dev, "use_pinctrl is %d\n",
		dev->use_pinctrl);
	PCIE_DBG_FS(dev, "use_19p2mhz_aux_clk is %d\n",
		dev->use_19p2mhz_aux_clk);
	PCIE_DBG_FS(dev, "user_suspend is %d\n",
		dev->user_suspend);
	PCIE_DBG_FS(dev, "num_ep: %d\n",
		dev->num_ep);
	PCIE_DBG_FS(dev, "num_active_ep: %d\n",
		dev->num_active_ep);
	PCIE_DBG_FS(dev, "pending_ep_reg: %s\n",
		dev->pending_ep_reg ? "true" : "false");
	PCIE_DBG_FS(dev, "phy_len is %d",
		dev->phy_len);
	PCIE_DBG_FS(dev, "disable_pc is %d",
		dev->disable_pc);
	PCIE_DBG_FS(dev, "l0s_supported is %s supported\n",
		dev->l0s_supported ? "" : "not");
	PCIE_DBG_FS(dev, "l1_supported is %s supported\n",
		dev->l1_supported ? "" : "not");
	PCIE_DBG_FS(dev, "l1ss_supported is %s supported\n",
		dev->l1ss_supported ? "" : "not");
	PCIE_DBG_FS(dev, "common_clk_en is %d\n",
		dev->common_clk_en);
	PCIE_DBG_FS(dev, "clk_power_manage_en is %d\n",
		dev->clk_power_manage_en);
	PCIE_DBG_FS(dev, "aux_clk_sync is %d\n",
		dev->aux_clk_sync);
	PCIE_DBG_FS(dev, "AER is %s enable\n",
		dev->aer_enable ? "" : "not");
	PCIE_DBG_FS(dev, "ext_ref_clk is %d\n",
		dev->ext_ref_clk);
	PCIE_DBG_FS(dev, "boot_option is 0x%x\n",
		dev->boot_option);
	PCIE_DBG_FS(dev, "phy_ver is %d\n",
		dev->phy_ver);
	PCIE_DBG_FS(dev, "drv_ready is %d\n",
		dev->drv_ready);
	PCIE_DBG_FS(dev, "linkdown_panic is %d\n",
		dev->linkdown_panic);
	PCIE_DBG_FS(dev, "the link is %s suspending\n",
		dev->suspending ? "" : "not");
	PCIE_DBG_FS(dev, "shadow is %s enabled\n",
		dev->shadow_en ? "" : "not");
	PCIE_DBG_FS(dev, "the power of RC is %s on\n",
		dev->power_on ? "" : "not");
	PCIE_DBG_FS(dev, "msi_gicm_addr: 0x%x\n",
		dev->msi_gicm_addr);
	PCIE_DBG_FS(dev, "msi_gicm_base: 0x%x\n",
		dev->msi_gicm_base);
	PCIE_DBG_FS(dev, "bus_client: %d\n",
		dev->bus_client);
	PCIE_DBG_FS(dev, "smmu does %s exist\n",
		dev->smmu_exist ? "" : "not");
	PCIE_DBG_FS(dev, "smmu_sid_base: 0x%x\n",
		dev->smmu_sid_base);
	PCIE_DBG_FS(dev, "n_fts: %d\n",
		dev->n_fts);
	PCIE_DBG_FS(dev, "ep_latency: %dms\n",
		dev->ep_latency);
	PCIE_DBG_FS(dev, "switch_latency: %dms\n",
		dev->switch_latency);
	PCIE_DBG_FS(dev, "wr_halt_size: 0x%x\n",
		dev->wr_halt_size);
	PCIE_DBG_FS(dev, "slv_addr_space_size: 0x%x\n",
		dev->slv_addr_space_size);
	PCIE_DBG_FS(dev, "phy_status_offset: 0x%x\n",
		dev->phy_status_offset);
	PCIE_DBG_FS(dev, "cpl_timeout: 0x%x\n",
		dev->cpl_timeout);
	PCIE_DBG_FS(dev, "current_bdf: 0x%x\n",
		dev->current_bdf);
	PCIE_DBG_FS(dev, "perst_delay_us_min: %dus\n",
		dev->perst_delay_us_min);
	PCIE_DBG_FS(dev, "perst_delay_us_max: %dus\n",
		dev->perst_delay_us_max);
	PCIE_DBG_FS(dev, "tlp_rd_size: 0x%x\n",
		dev->tlp_rd_size);
	PCIE_DBG_FS(dev, "rc_corr_counter: %lu\n",
		dev->rc_corr_counter);
	PCIE_DBG_FS(dev, "rc_non_fatal_counter: %lu\n",
		dev->rc_non_fatal_counter);
	PCIE_DBG_FS(dev, "rc_fatal_counter: %lu\n",
		dev->rc_fatal_counter);
	PCIE_DBG_FS(dev, "ep_corr_counter: %lu\n",
		dev->ep_corr_counter);
	PCIE_DBG_FS(dev, "ep_non_fatal_counter: %lu\n",
		dev->ep_non_fatal_counter);
	PCIE_DBG_FS(dev, "ep_fatal_counter: %lu\n",
		dev->ep_fatal_counter);
	PCIE_DBG_FS(dev, "linkdown_counter: %lu\n",
		dev->linkdown_counter);
	PCIE_DBG_FS(dev, "wake_counter: %lu\n",
		dev->wake_counter);
	PCIE_DBG_FS(dev, "max_link_speed: 0x%x\n",
		dev->max_link_speed);
	PCIE_DBG_FS(dev, "link_turned_on_counter: %lu\n",
		dev->link_turned_on_counter);
	PCIE_DBG_FS(dev, "link_turned_off_counter: %lu\n",
		dev->link_turned_off_counter);
}

static void msm_pcie_shadow_dump(struct msm_pcie_dev_t *dev, bool rc)
{
	int i, j;
	u32 val = 0;
	u32 *shadow;

	for (i = 0; i < MAX_DEVICE_NUM; i++) {
		if (!rc && !dev->pcidev_table[i].bdf)
			break;
		if (rc) {
			shadow = dev->rc_shadow;
		} else {
			shadow = dev->ep_shadow[i];
			PCIE_DBG_FS(dev, "PCIe Device: %02x:%02x.%01x\n",
				dev->pcidev_table[i].bdf >> 24,
				dev->pcidev_table[i].bdf >> 19 & 0x1f,
				dev->pcidev_table[i].bdf >> 16 & 0x07);
		}
		for (j = 0; j < PCIE_CONF_SPACE_DW; j++) {
			val = shadow[j];
			if (val != PCIE_CLEAR) {
				PCIE_DBG_FS(dev,
					"PCIe: shadow_dw[%d]:cfg 0x%x:0x%x\n",
					j, j * 4, val);
			}
		}
		if (rc)
			break;
	}
}

static void msm_pcie_sel_debug_testcase(struct msm_pcie_dev_t *dev,
					u32 testcase)
{
	u32 dbi_base_addr = dev->res[MSM_PCIE_RES_DM_CORE].resource->start;
	phys_addr_t loopback_lbar_phy =
		dev->res[MSM_PCIE_RES_DM_CORE].resource->start +
		LOOPBACK_BASE_ADDR_OFFSET;
	static uint32_t loopback_val = 0x1;
	static dma_addr_t loopback_ddr_phy;
	static uint32_t *loopback_ddr_vir;
	static void __iomem *loopback_lbar_vir;
	int ret, i;
	u32 base_sel_size = 0;

	switch (testcase) {
	case MSM_PCIE_OUTPUT_PCIE_INFO:
		PCIE_DBG_FS(dev, "\n\nPCIe: Status for RC%d:\n",
			dev->rc_idx);
		msm_pcie_show_status(dev);
		break;
	case MSM_PCIE_DISABLE_LINK:
		PCIE_DBG_FS(dev,
			"\n\nPCIe: RC%d: disable link\n\n", dev->rc_idx);
		ret = msm_pcie_pm_control(MSM_PCIE_SUSPEND, 0,
			dev->dev, NULL,
			MSM_PCIE_CONFIG_NO_CFG_RESTORE);
		if (ret)
			PCIE_DBG_FS(dev, "PCIe:%s:failed to disable link\n",
				__func__);
		else
			PCIE_DBG_FS(dev, "PCIe:%s:disabled link\n",
				__func__);
		break;
	case MSM_PCIE_ENABLE_LINK:
		PCIE_DBG_FS(dev,
			"\n\nPCIe: RC%d: enable link and recover config space\n\n",
			dev->rc_idx);
		ret = msm_pcie_pm_control(MSM_PCIE_RESUME, 0,
			dev->dev, NULL,
			MSM_PCIE_CONFIG_NO_CFG_RESTORE);
		if (ret)
			PCIE_DBG_FS(dev, "PCIe:%s:failed to enable link\n",
				__func__);
		else {
			PCIE_DBG_FS(dev, "PCIe:%s:enabled link\n", __func__);
			msm_pcie_recover_config(dev->dev);
		}
		break;
	case MSM_PCIE_DISABLE_ENABLE_LINK:
		PCIE_DBG_FS(dev,
			"\n\nPCIe: RC%d: disable and enable link then recover config space\n\n",
			dev->rc_idx);
		ret = msm_pcie_pm_control(MSM_PCIE_SUSPEND, 0,
			dev->dev, NULL,
			MSM_PCIE_CONFIG_NO_CFG_RESTORE);
		if (ret)
			PCIE_DBG_FS(dev, "PCIe:%s:failed to disable link\n",
				__func__);
		else
			PCIE_DBG_FS(dev, "PCIe:%s:disabled link\n", __func__);
		ret = msm_pcie_pm_control(MSM_PCIE_RESUME, 0,
			dev->dev, NULL,
			MSM_PCIE_CONFIG_NO_CFG_RESTORE);
		if (ret)
			PCIE_DBG_FS(dev, "PCIe:%s:failed to enable link\n",
				__func__);
		else {
			PCIE_DBG_FS(dev, "PCIe:%s:enabled link\n", __func__);
			msm_pcie_recover_config(dev->dev);
		}
		break;
	case MSM_PCIE_DUMP_SHADOW_REGISTER:
		PCIE_DBG_FS(dev,
			"\n\nPCIe: RC%d: dumping RC shadow registers\n",
			dev->rc_idx);
		msm_pcie_shadow_dump(dev, true);

		PCIE_DBG_FS(dev,
			"\n\nPCIe: RC%d: dumping EP shadow registers\n",
			dev->rc_idx);
		msm_pcie_shadow_dump(dev, false);
		break;
	case MSM_PCIE_DISABLE_L0S:
		PCIE_DBG_FS(dev, "\n\nPCIe: RC%d: disable L0s\n\n",
			dev->rc_idx);
		if (dev->link_status == MSM_PCIE_LINK_ENABLED) {
			struct pci_bus *bus, *c_bus;
			struct list_head *children = &dev->dev->bus->children;

			msm_pcie_config_l0s_disable(dev->dev, dev);

			list_for_each_entry_safe(bus, c_bus, children, node)
				pci_walk_bus(bus,
					&msm_pcie_config_l0s_disable, dev);
		}
		dev->l0s_supported = false;
		break;
	case MSM_PCIE_ENABLE_L0S:
		PCIE_DBG_FS(dev, "\n\nPCIe: RC%d: enable L0s\n\n",
			dev->rc_idx);
		dev->l0s_supported = true;
		if (dev->link_status == MSM_PCIE_LINK_ENABLED) {
			struct pci_bus *bus, *c_bus;
			struct list_head *children = &dev->dev->bus->children;

			list_for_each_entry_safe(bus, c_bus, children, node)
				pci_walk_bus(bus,
					&msm_pcie_config_l0s_enable, dev);

			msm_pcie_config_l0s_enable(dev->dev, dev);
		}
		break;
	case MSM_PCIE_DISABLE_L1:
		PCIE_DBG_FS(dev, "\n\nPCIe: RC%d: disable L1\n\n",
			dev->rc_idx);
		if (dev->link_status == MSM_PCIE_LINK_ENABLED) {
			struct pci_bus *bus, *c_bus;
			struct list_head *children = &dev->dev->bus->children;

			msm_pcie_config_l1_disable(dev->dev, dev);

			list_for_each_entry_safe(bus, c_bus, children, node)
				pci_walk_bus(bus,
					&msm_pcie_config_l1_disable, dev);
		}
		dev->l1_supported = false;
		break;
	case MSM_PCIE_ENABLE_L1:
		PCIE_DBG_FS(dev, "\n\nPCIe: RC%d: enable L1\n\n",
			dev->rc_idx);
		dev->l1_supported = true;
		if (dev->link_status == MSM_PCIE_LINK_ENABLED) {
			struct pci_bus *bus, *c_bus;
			struct list_head *children = &dev->dev->bus->children;

			list_for_each_entry_safe(bus, c_bus, children, node)
				pci_walk_bus(bus,
					&msm_pcie_config_l1_enable, dev);

			msm_pcie_config_l1_enable(dev->dev, dev);
		}
		break;
	case MSM_PCIE_DISABLE_L1SS:
		PCIE_DBG_FS(dev, "\n\nPCIe: RC%d: disable L1ss\n\n",
			dev->rc_idx);
		if (dev->link_status == MSM_PCIE_LINK_ENABLED) {
			struct pci_bus *bus, *c_bus;
			struct list_head *children = &dev->dev->bus->children;

			msm_pcie_config_l1ss_disable(dev->dev, dev);

			list_for_each_entry_safe(bus, c_bus, children, node)
				pci_walk_bus(bus,
					&msm_pcie_config_l1ss_disable, dev);
		}
		dev->l1ss_supported = false;
		break;
	case MSM_PCIE_ENABLE_L1SS:
		PCIE_DBG_FS(dev, "\n\nPCIe: RC%d: enable L1ss\n\n",
			dev->rc_idx);
		dev->l1ss_supported = true;
		if (dev->link_status == MSM_PCIE_LINK_ENABLED) {
			struct pci_bus *bus, *c_bus;
			struct list_head *children = &dev->dev->bus->children;

			list_for_each_entry_safe(bus, c_bus, children, node)
				pci_walk_bus(bus,
					&msm_pcie_config_l1ss_enable, dev);

			msm_pcie_config_l1ss_enable(dev->dev, dev);
		}
		break;
	case MSM_PCIE_ENUMERATION:
		PCIE_DBG_FS(dev, "\n\nPCIe: attempting to enumerate RC%d\n\n",
			dev->rc_idx);
		if (dev->enumerated)
			PCIE_DBG_FS(dev, "PCIe: RC%d is already enumerated\n",
				dev->rc_idx);
		else {
			if (!msm_pcie_enumerate(dev->rc_idx))
				PCIE_DBG_FS(dev,
					"PCIe: RC%d is successfully enumerated\n",
					dev->rc_idx);
			else
				PCIE_DBG_FS(dev,
					"PCIe: RC%d enumeration failed\n",
					dev->rc_idx);
		}
		break;
	case MSM_PCIE_READ_PCIE_REGISTER:
		PCIE_DBG_FS(dev,
			"\n\nPCIe: RC%d: read a PCIe register\n\n",
			dev->rc_idx);
		if (!base_sel) {
			PCIE_DBG_FS(dev, "Invalid base_sel: 0x%x\n", base_sel);
			break;
		}

		PCIE_DBG_FS(dev, "base: %s: 0x%pK\nwr_offset: 0x%x\n",
			dev->res[base_sel - 1].name,
			dev->res[base_sel - 1].base,
			wr_offset);

		base_sel_size = resource_size(dev->res[base_sel - 1].resource);

		if (wr_offset >  base_sel_size - 4 ||
			msm_pcie_check_align(dev, wr_offset)) {
			PCIE_DBG_FS(dev,
				"PCIe: RC%d: Invalid wr_offset: 0x%x. wr_offset should be no more than 0x%x\n",
				dev->rc_idx, wr_offset, base_sel_size - 4);
		} else {
			phys_addr_t wr_register =
				dev->res[MSM_PCIE_RES_DM_CORE].resource->start;

			wr_register += wr_offset;
			PCIE_DBG_FS(dev,
				"PCIe: RC%d: register: 0x%pa value: 0x%x\n",
				dev->rc_idx, &wr_register,
				readl_relaxed(dev->res[base_sel - 1].base +
					wr_offset));
		}

		break;
	case MSM_PCIE_WRITE_PCIE_REGISTER:
		PCIE_DBG_FS(dev,
			"\n\nPCIe: RC%d: writing a value to a register\n\n",
			dev->rc_idx);

		if (!base_sel) {
			PCIE_DBG_FS(dev, "Invalid base_sel: 0x%x\n", base_sel);
			break;
		}

		PCIE_DBG_FS(dev,
			"base: %s: 0x%pK\nwr_offset: 0x%x\nwr_mask: 0x%x\nwr_value: 0x%x\n",
			dev->res[base_sel - 1].name,
			dev->res[base_sel - 1].base,
			wr_offset, wr_mask, wr_value);

		base_sel_size = resource_size(dev->res[base_sel - 1].resource);

		if (wr_offset >  base_sel_size - 4 ||
			msm_pcie_check_align(dev, wr_offset))
			PCIE_DBG_FS(dev,
				"PCIe: RC%d: Invalid wr_offset: 0x%x. wr_offset should be no more than 0x%x\n",
				dev->rc_idx, wr_offset, base_sel_size - 4);
		else
			msm_pcie_write_reg_field(dev->res[base_sel - 1].base,
				wr_offset, wr_mask, wr_value);

		break;
	case MSM_PCIE_DUMP_PCIE_REGISTER_SPACE:
		if (!base_sel) {
			PCIE_DBG_FS(dev, "Invalid base_sel: 0x%x\n", base_sel);
			break;
		} else if (base_sel - 1 == MSM_PCIE_RES_PARF) {
			pcie_parf_dump(dev);
			break;
		} else if (base_sel - 1 == MSM_PCIE_RES_PHY) {
			pcie_phy_dump(dev);
			break;
		} else if (base_sel - 1 == MSM_PCIE_RES_CONF) {
			base_sel_size = 0x1000;
		} else {
			base_sel_size = resource_size(
				dev->res[base_sel - 1].resource);
		}

		PCIE_DBG_FS(dev, "\n\nPCIe: Dumping %s Registers for RC%d\n\n",
			dev->res[base_sel - 1].name, dev->rc_idx);

		for (i = 0; i < base_sel_size; i += 32) {
			PCIE_DBG_FS(dev,
			"0x%04x %08x %08x %08x %08x %08x %08x %08x %08x\n",
			i, readl_relaxed(dev->res[base_sel - 1].base + i),
			readl_relaxed(dev->res[base_sel - 1].base + (i + 4)),
			readl_relaxed(dev->res[base_sel - 1].base + (i + 8)),
			readl_relaxed(dev->res[base_sel - 1].base + (i + 12)),
			readl_relaxed(dev->res[base_sel - 1].base + (i + 16)),
			readl_relaxed(dev->res[base_sel - 1].base + (i + 20)),
			readl_relaxed(dev->res[base_sel - 1].base + (i + 24)),
			readl_relaxed(dev->res[base_sel - 1].base + (i + 28)));
		}
		break;
	case MSM_PCIE_ALLOCATE_DDR_MAP_LBAR:
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: Allocate 4K DDR memory and map LBAR.\n",
			dev->rc_idx);
		loopback_ddr_vir = dma_alloc_coherent(&dev->pdev->dev,
			(SZ_1K * sizeof(*loopback_ddr_vir)),
			&loopback_ddr_phy, GFP_KERNEL);
		if (!loopback_ddr_vir) {
			PCIE_DBG_FS(dev,
				"PCIe: RC%d: failed to dma_alloc_coherent.\n",
				dev->rc_idx);
		} else {
			PCIE_DBG_FS(dev,
				"PCIe: RC%d: VIR DDR memory address: 0x%pK\n",
				dev->rc_idx, loopback_ddr_vir);
			PCIE_DBG_FS(dev,
				"PCIe: RC%d: PHY DDR memory address: %pad\n",
				dev->rc_idx, &loopback_ddr_phy);
		}

		PCIE_DBG_FS(dev, "PCIe: RC%d: map LBAR: %pa\n",
			dev->rc_idx, &loopback_lbar_phy);
		loopback_lbar_vir = devm_ioremap(&dev->pdev->dev,
			loopback_lbar_phy, SZ_4K);
		if (!loopback_lbar_vir) {
			PCIE_DBG_FS(dev, "PCIe: RC%d: failed to map %pa\n",
				dev->rc_idx, &loopback_lbar_phy);
		} else {
			PCIE_DBG_FS(dev,
				"PCIe: RC%d: successfully mapped %pa to 0x%pK\n",
				dev->rc_idx, &loopback_lbar_phy,
				loopback_lbar_vir);
		}
		break;
	case MSM_PCIE_FREE_DDR_UNMAP_LBAR:
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: Release 4K DDR memory and unmap LBAR.\n",
			dev->rc_idx);

		if (loopback_ddr_vir) {
			dma_free_coherent(&dev->pdev->dev, SZ_4K,
				loopback_ddr_vir, loopback_ddr_phy);
			loopback_ddr_vir = NULL;
		}

		if (loopback_lbar_vir) {
			devm_iounmap(&dev->pdev->dev,
				loopback_lbar_vir);
			loopback_lbar_vir = NULL;
		}
		break;
	case MSM_PCIE_OUTPUT_DDR_LBAR_ADDRESS:
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: Print DDR and LBAR addresses.\n",
			dev->rc_idx);

		if (!loopback_ddr_vir || !loopback_lbar_vir) {
			PCIE_DBG_FS(dev,
				"PCIe: RC%d: DDR or LBAR address is not mapped\n",
				dev->rc_idx);
			break;
		}

		PCIE_DBG_FS(dev,
			"PCIe: RC%d: PHY DDR address: %pad\n",
			dev->rc_idx, &loopback_ddr_phy);
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: VIR DDR address: 0x%pK\n",
			dev->rc_idx, loopback_ddr_vir);
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: PHY LBAR address: %pa\n",
			dev->rc_idx, &loopback_lbar_phy);
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: VIR LBAR address: 0x%pK\n",
			dev->rc_idx, loopback_lbar_vir);
		break;
	case MSM_PCIE_CONFIGURE_LOOPBACK:
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: Configure Loopback.\n",
			dev->rc_idx);

		writel_relaxed(0x10000,
			dev->dm_core + PCIE20_GEN3_RELATED_REG);
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: 0x%x: 0x%x\n",
			dev->rc_idx,
			dbi_base_addr + PCIE20_GEN3_RELATED_REG,
			readl_relaxed(dev->dm_core +
				PCIE20_GEN3_RELATED_REG));

		writel_relaxed(0x80000001,
			dev->dm_core + PCIE20_PIPE_LOOPBACK_CONTROL);
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: 0x%x: 0x%x\n",
			dev->rc_idx,
			dbi_base_addr + PCIE20_PIPE_LOOPBACK_CONTROL,
			readl_relaxed(dev->dm_core +
				PCIE20_PIPE_LOOPBACK_CONTROL));

		writel_relaxed(0x00010124,
			dev->dm_core + PCIE20_PORT_LINK_CTRL_REG);
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: 0x%x: 0x%x\n",
			dev->rc_idx,
			dbi_base_addr + PCIE20_PORT_LINK_CTRL_REG,
			readl_relaxed(dev->dm_core +
				PCIE20_PORT_LINK_CTRL_REG));
		break;
	case MSM_PCIE_SETUP_LOOPBACK_IATU:
		PCIE_DBG_FS(dev, "PCIe: RC%d: Setup iATU.\n", dev->rc_idx);

		if (!loopback_ddr_vir) {
			PCIE_DBG_FS(dev,
				"PCIe: RC%d: DDR address is not mapped.\n",
				dev->rc_idx);
			break;
		}

		writel_relaxed(0x0, dev->dm_core + PCIE20_PLR_IATU_VIEWPORT);
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: PCIE20_PLR_IATU_VIEWPORT:\t0x%x: 0x%x\n",
			dev->rc_idx, dbi_base_addr + PCIE20_PLR_IATU_VIEWPORT,
			readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_VIEWPORT));

		writel_relaxed(0x0, dev->dm_core + PCIE20_PLR_IATU_CTRL1);
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: PCIE20_PLR_IATU_CTRL1:\t0x%x: 0x%x\n",
			dev->rc_idx, dbi_base_addr + PCIE20_PLR_IATU_CTRL1,
			readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_CTRL1));

		writel_relaxed(loopback_lbar_phy,
			dev->dm_core + PCIE20_PLR_IATU_LBAR);
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: PCIE20_PLR_IATU_LBAR:\t0x%x: 0x%x\n",
			dev->rc_idx, dbi_base_addr + PCIE20_PLR_IATU_LBAR,
			readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_LBAR));

		writel_relaxed(0x0, dev->dm_core + PCIE20_PLR_IATU_UBAR);
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: PCIE20_PLR_IATU_UBAR:\t0x%x: 0x%x\n",
			dev->rc_idx, dbi_base_addr + PCIE20_PLR_IATU_UBAR,
			readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_UBAR));

		writel_relaxed(loopback_lbar_phy + 0xfff,
			dev->dm_core + PCIE20_PLR_IATU_LAR);
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: PCIE20_PLR_IATU_LAR:\t0x%x: 0x%x\n",
			dev->rc_idx, dbi_base_addr + PCIE20_PLR_IATU_LAR,
			readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_LAR));

		writel_relaxed(loopback_ddr_phy,
			dev->dm_core + PCIE20_PLR_IATU_LTAR);
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: PCIE20_PLR_IATU_LTAR:\t0x%x: 0x%x\n",
			dev->rc_idx, dbi_base_addr + PCIE20_PLR_IATU_LTAR,
			readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_LTAR));

		writel_relaxed(0, dev->dm_core + PCIE20_PLR_IATU_UTAR);
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: PCIE20_PLR_IATU_UTAR:\t0x%x: 0x%x\n",
			dev->rc_idx, dbi_base_addr + PCIE20_PLR_IATU_UTAR,
			readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_UTAR));

		writel_relaxed(0x80000000,
			dev->dm_core + PCIE20_PLR_IATU_CTRL2);
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: PCIE20_PLR_IATU_CTRL2:\t0x%x: 0x%x\n",
			dev->rc_idx, dbi_base_addr + PCIE20_PLR_IATU_CTRL2,
			readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_CTRL2));
		break;
	case MSM_PCIE_READ_DDR:
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: Read DDR values.\n",
			dev->rc_idx);

		if (!loopback_ddr_vir) {
			PCIE_DBG_FS(dev,
				"PCIe: RC%d: DDR is not mapped\n",
				dev->rc_idx);
			break;
		}

		for (i = 0; i < SZ_1K; i += 8) {
			PCIE_DBG_FS(dev,
				"0x%04x %08x %08x %08x %08x %08x %08x %08x %08x\n",
				i,
				loopback_ddr_vir[i],
				loopback_ddr_vir[i + 1],
				loopback_ddr_vir[i + 2],
				loopback_ddr_vir[i + 3],
				loopback_ddr_vir[i + 4],
				loopback_ddr_vir[i + 5],
				loopback_ddr_vir[i + 6],
				loopback_ddr_vir[i + 7]);
		}
		break;
	case MSM_PCIE_READ_LBAR:
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: Read LBAR values.\n",
			dev->rc_idx);

		if (!loopback_lbar_vir) {
			PCIE_DBG_FS(dev,
				"PCIe: RC%d: LBAR address is not mapped\n",
				dev->rc_idx);
			break;
		}

		for (i = 0; i < SZ_4K; i += 32) {
			PCIE_DBG_FS(dev,
				"0x%04x %08x %08x %08x %08x %08x %08x %08x %08x\n",
				i,
				readl_relaxed(loopback_lbar_vir + i),
				readl_relaxed(loopback_lbar_vir + (i + 4)),
				readl_relaxed(loopback_lbar_vir + (i + 8)),
				readl_relaxed(loopback_lbar_vir + (i + 12)),
				readl_relaxed(loopback_lbar_vir + (i + 16)),
				readl_relaxed(loopback_lbar_vir + (i + 20)),
				readl_relaxed(loopback_lbar_vir + (i + 24)),
				readl_relaxed(loopback_lbar_vir + (i + 28)));
		}
		break;
	case MSM_PCIE_WRITE_DDR:
		PCIE_DBG_FS(dev, "PCIe: RC%d: Write 0x%x to DDR.\n",
			dev->rc_idx, loopback_val);

		if (!loopback_ddr_vir) {
			PCIE_DBG_FS(dev,
				"PCIe: RC%d: DDR address is not mapped\n",
				dev->rc_idx);
			break;
		}

		memset(loopback_ddr_vir, loopback_val,
			(SZ_1K * sizeof(*loopback_ddr_vir)));

		if (unlikely(loopback_val == UINT_MAX))
			loopback_val = 1;
		else
			loopback_val++;
		break;
	case MSM_PCIE_WRITE_LBAR:
		PCIE_DBG_FS(dev, "PCIe: RC%d: Write 0x%x to LBAR.\n",
			dev->rc_idx, loopback_val);

		if (!loopback_lbar_vir) {
			PCIE_DBG_FS(dev,
				"PCIe: RC%d: LBAR address is not mapped\n",
				dev->rc_idx);
			break;
		}

		for (i = 0; i < SZ_4K; i += 32) {
			writel_relaxed(loopback_val,
				loopback_lbar_vir + i),
			writel_relaxed(loopback_val,
				loopback_lbar_vir + (i + 4)),
			writel_relaxed(loopback_val,
				loopback_lbar_vir + (i + 8)),
			writel_relaxed(loopback_val,
				loopback_lbar_vir + (i + 12)),
			writel_relaxed(loopback_val,
				loopback_lbar_vir + (i + 16)),
			writel_relaxed(loopback_val,
				loopback_lbar_vir + (i + 20)),
			writel_relaxed(loopback_val,
				loopback_lbar_vir + (i + 24)),
			writel_relaxed(loopback_val,
				loopback_lbar_vir + (i + 28));
		}

		if (unlikely(loopback_val == UINT_MAX))
			loopback_val = 1;
		else
			loopback_val++;
		break;
	case MSM_PCIE_DISABLE_AER:
		PCIE_DBG_FS(dev, "\n\nPCIe: RC%d: clear AER enable flag\n\n",
			dev->rc_idx);
		dev->aer_enable = false;
		break;
	case MSM_PCIE_ENABLE_AER:
		PCIE_DBG_FS(dev, "\n\nPCIe: RC%d: set AER enable flag\n\n",
			dev->rc_idx);
		dev->aer_enable = true;
		break;
	case MSM_PCIE_GPIO_STATUS:
		PCIE_DBG_FS(dev, "\n\nPCIe: RC%d: PERST and WAKE status\n\n",
			dev->rc_idx);
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: PERST: gpio%u value: %d\n",
			dev->rc_idx, dev->gpio[MSM_PCIE_GPIO_PERST].num,
			gpio_get_value(dev->gpio[MSM_PCIE_GPIO_PERST].num));
		PCIE_DBG_FS(dev,
			"PCIe: RC%d: WAKE: gpio%u value: %d\n",
			dev->rc_idx, dev->gpio[MSM_PCIE_GPIO_WAKE].num,
			gpio_get_value(dev->gpio[MSM_PCIE_GPIO_WAKE].num));
		break;
	case MSM_PCIE_ASSERT_PERST:
		PCIE_DBG_FS(dev, "\n\nPCIe: RC%d: assert PERST\n\n",
			dev->rc_idx);
		gpio_set_value(dev->gpio[MSM_PCIE_GPIO_PERST].num,
					dev->gpio[MSM_PCIE_GPIO_PERST].on);
		usleep_range(dev->perst_delay_us_min, dev->perst_delay_us_max);
		break;
	case MSM_PCIE_DEASSERT_PERST:
		PCIE_DBG_FS(dev, "\n\nPCIe: RC%d: de-assert PERST\n\n",
			dev->rc_idx);
		gpio_set_value(dev->gpio[MSM_PCIE_GPIO_PERST].num,
					1 - dev->gpio[MSM_PCIE_GPIO_PERST].on);
		usleep_range(dev->perst_delay_us_min, dev->perst_delay_us_max);
		break;
	case MSM_PCIE_KEEP_RESOURCES_ON:
		PCIE_DBG_FS(dev,
			"\n\nPCIe: RC%d: set keep resources on flag\n\n",
			dev->rc_idx);
		msm_pcie_keep_resources_on |= BIT(dev->rc_idx);
		break;
	case MSM_PCIE_FORCE_GEN1:
		PCIE_DBG_FS(dev, "\n\nPCIe: RC%d: set force gen1 flag\n\n",
			dev->rc_idx);
		msm_pcie_force_gen1 |= BIT(dev->rc_idx);
		break;
	default:
		PCIE_DBG_FS(dev, "Invalid testcase: %d.\n", testcase);
		break;
	}
}

int msm_pcie_debug_info(struct pci_dev *dev, u32 option, u32 base,
			u32 offset, u32 mask, u32 value)
{
	int ret = 0;
	struct msm_pcie_dev_t *pdev = NULL;

	if (!dev) {
		pr_err("PCIe: the input pci dev is NULL.\n");
		return -ENODEV;
	}

	if (option == MSM_PCIE_READ_PCIE_REGISTER ||
		option == MSM_PCIE_WRITE_PCIE_REGISTER ||
		option == MSM_PCIE_DUMP_PCIE_REGISTER_SPACE) {
		if (!base || base >= MSM_PCIE_MAX_RES) {
			PCIE_DBG_FS(pdev, "Invalid base_sel: 0x%x\n", base);
			PCIE_DBG_FS(pdev,
				"PCIe: base_sel is still 0x%x\n", base_sel);
			return -EINVAL;
		}

		base_sel = base;
		PCIE_DBG_FS(pdev, "PCIe: base_sel is now 0x%x\n", base_sel);

		if (option == MSM_PCIE_READ_PCIE_REGISTER ||
			option == MSM_PCIE_WRITE_PCIE_REGISTER) {
			wr_offset = offset;
			wr_mask = mask;
			wr_value = value;

			PCIE_DBG_FS(pdev,
				"PCIe: wr_offset is now 0x%x\n", wr_offset);
			PCIE_DBG_FS(pdev,
				"PCIe: wr_mask is now 0x%x\n", wr_mask);
			PCIE_DBG_FS(pdev,
				"PCIe: wr_value is now 0x%x\n", wr_value);
		}
	}

	pdev = PCIE_BUS_PRIV_DATA(dev->bus);
	rc_sel = BIT(pdev->rc_idx);

	msm_pcie_sel_debug_testcase(pdev, option);

	return ret;
}
EXPORT_SYMBOL(msm_pcie_debug_info);

#ifdef CONFIG_SYSFS
static ssize_t msm_pcie_enumerate_store(struct device *dev,
					struct device_attribute *attr,
					const char *buf, size_t count)
{
	struct msm_pcie_dev_t *pcie_dev = (struct msm_pcie_dev_t *)
						dev_get_drvdata(dev);

	if (pcie_dev)
		msm_pcie_enumerate(pcie_dev->rc_idx);

	return count;
}

static DEVICE_ATTR(enumerate, 0200, NULL, msm_pcie_enumerate_store);

static void msm_pcie_sysfs_init(struct msm_pcie_dev_t *dev)
{
	int ret;

	ret = device_create_file(&dev->pdev->dev, &dev_attr_enumerate);
	if (ret)
		PCIE_DBG_FS(dev,
			"RC%d: failed to create sysfs enumerate node\n",
			dev->rc_idx);
}

static void msm_pcie_sysfs_exit(struct msm_pcie_dev_t *dev)
{
	if (dev->pdev)
		device_remove_file(&dev->pdev->dev, &dev_attr_enumerate);
}
#else
static void msm_pcie_sysfs_init(struct msm_pcie_dev_t *dev)
{
}

static void msm_pcie_sysfs_exit(struct msm_pcie_dev_t *dev)
{
}
#endif

#ifdef CONFIG_DEBUG_FS
static struct dentry *dent_msm_pcie;
static struct dentry *dfile_rc_sel;
static struct dentry *dfile_case;
static struct dentry *dfile_base_sel;
static struct dentry *dfile_linkdown_panic;
static struct dentry *dfile_wr_offset;
static struct dentry *dfile_wr_mask;
static struct dentry *dfile_wr_value;
static struct dentry *dfile_boot_option;
static struct dentry *dfile_aer_enable;
static struct dentry *dfile_corr_counter_limit;

static u32 rc_sel_max;

static int msm_pcie_debugfs_parse_input(const char __user *buf,
					size_t count, unsigned int *data)
{
	unsigned long ret;
	char *str, *str_temp;

	str = kmalloc(count + 1, GFP_KERNEL);
	if (!str)
		return -ENOMEM;

	ret = copy_from_user(str, buf, count);
	if (ret) {
		kfree(str);
		return -EFAULT;
	}

	str[count] = 0;
	str_temp = str;

	ret = get_option(&str_temp, data);
	kfree(str);
	if (ret != 1)
		return -EINVAL;

	return 0;
}

static int msm_pcie_debugfs_case_show(struct seq_file *m, void *v)
{
	int i;

	for (i = 0; i < MSM_PCIE_MAX_DEBUGFS_OPTION; i++)
		seq_printf(m, "\t%d:\t %s\n", i,
			msm_pcie_debugfs_option_desc[i]);

	return 0;
}

static int msm_pcie_debugfs_case_open(struct inode *inode, struct file *file)
{
	return single_open(file, msm_pcie_debugfs_case_show, NULL);
}

static ssize_t msm_pcie_debugfs_case_select(struct file *file,
				const char __user *buf,
				size_t count, loff_t *ppos)
{
	int i, ret;
	unsigned int testcase = 0;

	ret = msm_pcie_debugfs_parse_input(buf, count, &testcase);
	if (ret)
		return ret;

	pr_alert("PCIe: TEST: %d\n", testcase);

	for (i = 0; i < MAX_RC_NUM; i++) {
		if (rc_sel & BIT(i))
			msm_pcie_sel_debug_testcase(&msm_pcie_dev[i], testcase);
	}

	return count;
}

static const struct file_operations msm_pcie_debugfs_case_ops = {
	.open = msm_pcie_debugfs_case_open,
	.release = single_release,
	.read = seq_read,
	.write = msm_pcie_debugfs_case_select,
};

static ssize_t msm_pcie_debugfs_rc_select(struct file *file,
				const char __user *buf,
				size_t count, loff_t *ppos)
{
	int i, ret;
	u32 new_rc_sel = 0;

	ret = msm_pcie_debugfs_parse_input(buf, count, &new_rc_sel);
	if (ret)
		return ret;

	if ((!new_rc_sel) || (new_rc_sel > rc_sel_max)) {
		pr_alert("PCIe: invalid value for rc_sel: 0x%x\n", new_rc_sel);
		pr_alert("PCIe: rc_sel is still 0x%x\n", rc_sel ? rc_sel : 0x1);
	} else {
		rc_sel = new_rc_sel;
		pr_alert("PCIe: rc_sel is now: 0x%x\n", rc_sel);
	}

	pr_alert("PCIe: the following RC(s) will be tested:\n");
	for (i = 0; i < MAX_RC_NUM; i++)
		if (rc_sel & BIT(i))
			pr_alert("RC %d\n", i);

	return count;
}

static const struct file_operations msm_pcie_debugfs_rc_select_ops = {
	.write = msm_pcie_debugfs_rc_select,
};

static ssize_t msm_pcie_debugfs_base_select(struct file *file,
				const char __user *buf,
				size_t count, loff_t *ppos)
{
	int ret;
	u32 new_base_sel = 0;

	ret = msm_pcie_debugfs_parse_input(buf, count, &new_base_sel);
	if (ret)
		return ret;

	if (!new_base_sel || new_base_sel > MSM_PCIE_MAX_RES) {
		pr_alert("PCIe: invalid value for base_sel: 0x%x\n",
			new_base_sel);
		pr_alert("PCIe: base_sel is still 0x%x\n", base_sel);
	} else {
		base_sel = new_base_sel;
		pr_alert("PCIe: base_sel is now 0x%x\n", base_sel);
		pr_alert("%s\n", msm_pcie_res_info[base_sel - 1].name);
	}

	return count;
}

static const struct file_operations msm_pcie_debugfs_base_select_ops = {
	.write = msm_pcie_debugfs_base_select,
};

static ssize_t msm_pcie_debugfs_linkdown_panic(struct file *file,
				const char __user *buf,
				size_t count, loff_t *ppos)
{
	int i, ret;
	u32 new_linkdown_panic = 0;

	ret = msm_pcie_debugfs_parse_input(buf, count, &new_linkdown_panic);
	if (ret)
		return ret;

	new_linkdown_panic = !!new_linkdown_panic;

	for (i = 0; i < MAX_RC_NUM; i++) {
		if (rc_sel & BIT(i)) {
			msm_pcie_dev[i].linkdown_panic =
				new_linkdown_panic;
			PCIE_DBG_FS(&msm_pcie_dev[i],
				"PCIe: RC%d: linkdown_panic is now %d\n",
				i, msm_pcie_dev[i].linkdown_panic);
		}
	}

	return count;
}

static const struct file_operations msm_pcie_debugfs_linkdown_panic_ops = {
	.write = msm_pcie_debugfs_linkdown_panic,
};

static ssize_t msm_pcie_debugfs_wr_offset(struct file *file,
				const char __user *buf,
				size_t count, loff_t *ppos)
{
	int ret;

	wr_offset = 0;

	ret = msm_pcie_debugfs_parse_input(buf, count, &wr_offset);
	if (ret)
		return ret;

	pr_alert("PCIe: wr_offset is now 0x%x\n", wr_offset);

	return count;
}

static const struct file_operations msm_pcie_debugfs_wr_offset_ops = {
	.write = msm_pcie_debugfs_wr_offset,
};

static ssize_t msm_pcie_debugfs_wr_mask(struct file *file,
				const char __user *buf,
				size_t count, loff_t *ppos)
{
	int ret;

	wr_mask = 0;

	ret = msm_pcie_debugfs_parse_input(buf, count, &wr_mask);
	if (ret)
		return ret;

	pr_alert("PCIe: wr_mask is now 0x%x\n", wr_mask);

	return count;
}

static const struct file_operations msm_pcie_debugfs_wr_mask_ops = {
	.write = msm_pcie_debugfs_wr_mask,
};
static ssize_t msm_pcie_debugfs_wr_value(struct file *file,
				const char __user *buf,
				size_t count, loff_t *ppos)
{
	int ret;

	wr_value = 0;

	ret = msm_pcie_debugfs_parse_input(buf, count, &wr_value);
	if (ret)
		return ret;

	pr_alert("PCIe: wr_value is now 0x%x\n", wr_value);

	return count;
}

static const struct file_operations msm_pcie_debugfs_wr_value_ops = {
	.write = msm_pcie_debugfs_wr_value,
};

static ssize_t msm_pcie_debugfs_boot_option(struct file *file,
				const char __user *buf,
				size_t count, loff_t *ppos)
{
	int i, ret;
	u32 new_boot_option = 0;

	ret = msm_pcie_debugfs_parse_input(buf, count, &new_boot_option);
	if (ret)
		return ret;

	if (new_boot_option <= (BIT(0) | BIT(1))) {
		for (i = 0; i < MAX_RC_NUM; i++) {
			if (rc_sel & BIT(i)) {
				msm_pcie_dev[i].boot_option = new_boot_option;
				PCIE_DBG_FS(&msm_pcie_dev[i],
					"PCIe: RC%d: boot_option is now 0x%x\n",
					i, msm_pcie_dev[i].boot_option);
			}
		}
	} else {
		pr_err("PCIe: Invalid input for boot_option: 0x%x.\n",
			new_boot_option);
	}

	return count;
}

static const struct file_operations msm_pcie_debugfs_boot_option_ops = {
	.write = msm_pcie_debugfs_boot_option,
};

static ssize_t msm_pcie_debugfs_aer_enable(struct file *file,
				const char __user *buf,
				size_t count, loff_t *ppos)
{
	int i, ret;
	u32 new_aer_enable = 0;

	ret = msm_pcie_debugfs_parse_input(buf, count, &new_aer_enable);
	if (ret)
		return ret;

	new_aer_enable = !!new_aer_enable;

	for (i = 0; i < MAX_RC_NUM; i++) {
		if (rc_sel & BIT(i)) {
			msm_pcie_dev[i].aer_enable = new_aer_enable;
			PCIE_DBG_FS(&msm_pcie_dev[i],
				"PCIe: RC%d: aer_enable is now %d\n",
				i, msm_pcie_dev[i].aer_enable);

			msm_pcie_write_mask(msm_pcie_dev[i].dm_core +
					PCIE20_BRIDGE_CTRL,
					new_aer_enable ? 0 : BIT(16),
					new_aer_enable ? BIT(16) : 0);

			PCIE_DBG_FS(&msm_pcie_dev[i],
				"RC%d: PCIE20_BRIDGE_CTRL: 0x%x\n", i,
				readl_relaxed(msm_pcie_dev[i].dm_core +
					PCIE20_BRIDGE_CTRL));
		}
	}

	return count;
}

static const struct file_operations msm_pcie_debugfs_aer_enable_ops = {
	.write = msm_pcie_debugfs_aer_enable,
};

static ssize_t msm_pcie_debugfs_corr_counter_limit(struct file *file,
				const char __user *buf,
				size_t count, loff_t *ppos)
{
	int ret;

	corr_counter_limit = 0;

	ret = msm_pcie_debugfs_parse_input(buf, count, &corr_counter_limit);
	if (ret)
		return ret;

	pr_info("PCIe: corr_counter_limit is now %u\n", corr_counter_limit);

	return count;
}

static const struct file_operations msm_pcie_debugfs_corr_counter_limit_ops = {
	.write = msm_pcie_debugfs_corr_counter_limit,
};

static void msm_pcie_debugfs_init(void)
{
	rc_sel_max = (0x1 << MAX_RC_NUM) - 1;
	wr_mask = 0xffffffff;

	dent_msm_pcie = debugfs_create_dir("pci-msm", NULL);
	if (IS_ERR(dent_msm_pcie)) {
		pr_err("PCIe: fail to create the folder for debug_fs.\n");
		return;
	}

	dfile_rc_sel = debugfs_create_file("rc_sel", 0664,
					dent_msm_pcie, NULL,
					&msm_pcie_debugfs_rc_select_ops);
	if (!dfile_rc_sel || IS_ERR(dfile_rc_sel)) {
		pr_err("PCIe: fail to create the file for debug_fs rc_sel.\n");
		goto rc_sel_error;
	}

	dfile_case = debugfs_create_file("case", 0664,
					dent_msm_pcie, NULL,
					&msm_pcie_debugfs_case_ops);
	if (!dfile_case || IS_ERR(dfile_case)) {
		pr_err("PCIe: fail to create the file for debug_fs case.\n");
		goto case_error;
	}

	dfile_base_sel = debugfs_create_file("base_sel", 0664,
					dent_msm_pcie, NULL,
					&msm_pcie_debugfs_base_select_ops);
	if (!dfile_base_sel || IS_ERR(dfile_base_sel)) {
		pr_err("PCIe: fail to create the file for debug_fs base_sel.\n");
		goto base_sel_error;
	}

	dfile_linkdown_panic = debugfs_create_file("linkdown_panic", 0644,
					dent_msm_pcie, NULL,
					&msm_pcie_debugfs_linkdown_panic_ops);
	if (!dfile_linkdown_panic || IS_ERR(dfile_linkdown_panic)) {
		pr_err("PCIe: fail to create the file for debug_fs linkdown_panic.\n");
		goto linkdown_panic_error;
	}

	dfile_wr_offset = debugfs_create_file("wr_offset", 0664,
					dent_msm_pcie, NULL,
					&msm_pcie_debugfs_wr_offset_ops);
	if (!dfile_wr_offset || IS_ERR(dfile_wr_offset)) {
		pr_err("PCIe: fail to create the file for debug_fs wr_offset.\n");
		goto wr_offset_error;
	}

	dfile_wr_mask = debugfs_create_file("wr_mask", 0664,
					dent_msm_pcie, NULL,
					&msm_pcie_debugfs_wr_mask_ops);
	if (!dfile_wr_mask || IS_ERR(dfile_wr_mask)) {
		pr_err("PCIe: fail to create the file for debug_fs wr_mask.\n");
		goto wr_mask_error;
	}

	dfile_wr_value = debugfs_create_file("wr_value", 0664,
					dent_msm_pcie, NULL,
					&msm_pcie_debugfs_wr_value_ops);
	if (!dfile_wr_value || IS_ERR(dfile_wr_value)) {
		pr_err("PCIe: fail to create the file for debug_fs wr_value.\n");
		goto wr_value_error;
	}

	dfile_boot_option = debugfs_create_file("boot_option", 0664,
					dent_msm_pcie, NULL,
					&msm_pcie_debugfs_boot_option_ops);
	if (!dfile_boot_option || IS_ERR(dfile_boot_option)) {
		pr_err("PCIe: fail to create the file for debug_fs boot_option.\n");
		goto boot_option_error;
	}

	dfile_aer_enable = debugfs_create_file("aer_enable", 0664,
					dent_msm_pcie, NULL,
					&msm_pcie_debugfs_aer_enable_ops);
	if (!dfile_aer_enable || IS_ERR(dfile_aer_enable)) {
		pr_err("PCIe: fail to create the file for debug_fs aer_enable.\n");
		goto aer_enable_error;
	}

	dfile_corr_counter_limit = debugfs_create_file("corr_counter_limit",
				0664, dent_msm_pcie, NULL,
				&msm_pcie_debugfs_corr_counter_limit_ops);
	if (!dfile_corr_counter_limit || IS_ERR(dfile_corr_counter_limit)) {
		pr_err("PCIe: fail to create the file for debug_fs corr_counter_limit.\n");
		goto corr_counter_limit_error;
	}
	return;

corr_counter_limit_error:
	debugfs_remove(dfile_aer_enable);
aer_enable_error:
	debugfs_remove(dfile_boot_option);
boot_option_error:
	debugfs_remove(dfile_wr_value);
wr_value_error:
	debugfs_remove(dfile_wr_mask);
wr_mask_error:
	debugfs_remove(dfile_wr_offset);
wr_offset_error:
	debugfs_remove(dfile_linkdown_panic);
linkdown_panic_error:
	debugfs_remove(dfile_base_sel);
base_sel_error:
	debugfs_remove(dfile_case);
case_error:
	debugfs_remove(dfile_rc_sel);
rc_sel_error:
	debugfs_remove(dent_msm_pcie);
}

static void msm_pcie_debugfs_exit(void)
{
	debugfs_remove(dfile_rc_sel);
	debugfs_remove(dfile_case);
	debugfs_remove(dfile_base_sel);
	debugfs_remove(dfile_linkdown_panic);
	debugfs_remove(dfile_wr_offset);
	debugfs_remove(dfile_wr_mask);
	debugfs_remove(dfile_wr_value);
	debugfs_remove(dfile_boot_option);
	debugfs_remove(dfile_aer_enable);
	debugfs_remove(dfile_corr_counter_limit);
}
#else
static void msm_pcie_debugfs_init(void)
{
}

static void msm_pcie_debugfs_exit(void)
{
}
#endif

static inline int msm_pcie_is_link_up(struct msm_pcie_dev_t *dev)
{
	return readl_relaxed(dev->dm_core +
			PCIE20_CAP_LINKCTRLSTATUS) & BIT(29);
}

/**
 * msm_pcie_iatu_config - configure outbound address translation region
 * @dev: root commpex
 * @nr: region number
 * @type: target transaction type, see PCIE20_CTRL1_TYPE_xxx
 * @host_addr: - region start address on host
 * @host_end: - region end address (low 32 bit) on host,
 *	upper 32 bits are same as for @host_addr
 * @target_addr: - region start address on target
 */
static void msm_pcie_iatu_config(struct msm_pcie_dev_t *dev, int nr, u8 type,
				unsigned long host_addr, u32 host_end,
				unsigned long target_addr)
{
	void __iomem *iatu_base = dev->iatu ? dev->iatu : dev->dm_core;

	u32 iatu_viewport_offset;
	u32 iatu_ctrl1_offset;
	u32 iatu_ctrl2_offset;
	u32 iatu_lbar_offset;
	u32 iatu_ubar_offset;
	u32 iatu_lar_offset;
	u32 iatu_ltar_offset;
	u32 iatu_utar_offset;

	if (dev->iatu) {
		iatu_viewport_offset = 0;
		iatu_ctrl1_offset = PCIE_IATU_CTRL1(nr);
		iatu_ctrl2_offset = PCIE_IATU_CTRL2(nr);
		iatu_lbar_offset = PCIE_IATU_LBAR(nr);
		iatu_ubar_offset = PCIE_IATU_UBAR(nr);
		iatu_lar_offset = PCIE_IATU_LAR(nr);
		iatu_ltar_offset = PCIE_IATU_LTAR(nr);
		iatu_utar_offset = PCIE_IATU_UTAR(nr);
	} else {
		iatu_viewport_offset = PCIE20_PLR_IATU_VIEWPORT;
		iatu_ctrl1_offset = PCIE20_PLR_IATU_CTRL1;
		iatu_ctrl2_offset = PCIE20_PLR_IATU_CTRL2;
		iatu_lbar_offset = PCIE20_PLR_IATU_LBAR;
		iatu_ubar_offset = PCIE20_PLR_IATU_UBAR;
		iatu_lar_offset = PCIE20_PLR_IATU_LAR;
		iatu_ltar_offset = PCIE20_PLR_IATU_LTAR;
		iatu_utar_offset = PCIE20_PLR_IATU_UTAR;
	}

	if (dev->shadow_en && iatu_viewport_offset) {
		dev->rc_shadow[PCIE20_PLR_IATU_VIEWPORT / 4] =
			nr;
		dev->rc_shadow[PCIE20_PLR_IATU_CTRL1 / 4] =
			type;
		dev->rc_shadow[PCIE20_PLR_IATU_LBAR / 4] =
			lower_32_bits(host_addr);
		dev->rc_shadow[PCIE20_PLR_IATU_UBAR / 4] =
			upper_32_bits(host_addr);
		dev->rc_shadow[PCIE20_PLR_IATU_LAR / 4] =
			host_end;
		dev->rc_shadow[PCIE20_PLR_IATU_LTAR / 4] =
			lower_32_bits(target_addr);
		dev->rc_shadow[PCIE20_PLR_IATU_UTAR / 4] =
			upper_32_bits(target_addr);
		dev->rc_shadow[PCIE20_PLR_IATU_CTRL2 / 4] =
			BIT(31);
	}

	/* select region */
	if (iatu_viewport_offset) {
		writel_relaxed(nr, iatu_base + iatu_viewport_offset);
		/* ensure that hardware locks it */
		wmb();
	}

	/* switch off region before changing it */
	writel_relaxed(0, iatu_base + iatu_ctrl2_offset);
	/* and wait till it propagates to the hardware */
	wmb();

	writel_relaxed(type, iatu_base + iatu_ctrl1_offset);
	writel_relaxed(lower_32_bits(host_addr),
		       iatu_base + iatu_lbar_offset);
	writel_relaxed(upper_32_bits(host_addr),
		       iatu_base + iatu_ubar_offset);
	writel_relaxed(host_end, iatu_base + iatu_lar_offset);
	writel_relaxed(lower_32_bits(target_addr),
		       iatu_base + iatu_ltar_offset);
	writel_relaxed(upper_32_bits(target_addr),
		       iatu_base + iatu_utar_offset);
	/* ensure that changes propagated to the hardware */
	wmb();
	writel_relaxed(BIT(31), iatu_base + iatu_ctrl2_offset);

	/* ensure that changes propagated to the hardware */
	wmb();

	if (dev->enumerated) {
		PCIE_DBG2(dev, "IATU for Endpoint %02x:%02x.%01x\n",
			dev->pcidev_table[nr].bdf >> 24,
			dev->pcidev_table[nr].bdf >> 19 & 0x1f,
			dev->pcidev_table[nr].bdf >> 16 & 0x07);
		if (iatu_viewport_offset)
			PCIE_DBG2(dev, "IATU_VIEWPORT:0x%x\n",
				readl_relaxed(dev->dm_core +
					PCIE20_PLR_IATU_VIEWPORT));
		PCIE_DBG2(dev, "IATU_CTRL1:0x%x\n",
			readl_relaxed(iatu_base + iatu_ctrl1_offset));
		PCIE_DBG2(dev, "IATU_LBAR:0x%x\n",
			readl_relaxed(iatu_base + iatu_lbar_offset));
		PCIE_DBG2(dev, "IATU_UBAR:0x%x\n",
			readl_relaxed(iatu_base + iatu_ubar_offset));
		PCIE_DBG2(dev, "IATU_LAR:0x%x\n",
			readl_relaxed(iatu_base + iatu_lar_offset));
		PCIE_DBG2(dev, "IATU_LTAR:0x%x\n",
			readl_relaxed(iatu_base + iatu_ltar_offset));
		PCIE_DBG2(dev, "IATU_UTAR:0x%x\n",
			readl_relaxed(iatu_base + iatu_utar_offset));
		PCIE_DBG2(dev, "IATU_CTRL2:0x%x\n\n",
			readl_relaxed(iatu_base + iatu_ctrl2_offset));
	}
}

/**
 * msm_pcie_cfg_bdf - configure for config access
 * @dev: root commpex
 * @bus: PCI bus number
 * @devfn: PCI dev and function number
 *
 * Remap if required region 0 for config access of proper type
 * (CFG0 for bus 1, CFG1 for other buses)
 * Cache current device bdf for speed-up
 */
static void msm_pcie_cfg_bdf(struct msm_pcie_dev_t *dev, u8 bus, u8 devfn)
{
	struct resource *axi_conf = dev->res[MSM_PCIE_RES_CONF].resource;
	u32 bdf  = BDF_OFFSET(bus, devfn);
	u8 type = bus == 1 ? PCIE20_CTRL1_TYPE_CFG0 : PCIE20_CTRL1_TYPE_CFG1;

	if (dev->current_bdf == bdf)
		return;

	msm_pcie_iatu_config(dev, 0, type,
			axi_conf->start,
			axi_conf->start + SZ_4K - 1,
			bdf);

	dev->current_bdf = bdf;
}

static inline void msm_pcie_save_shadow(struct msm_pcie_dev_t *dev,
					u32 word_offset, u32 wr_val,
					u32 bdf, bool rc)
{
	int i, j;
	u32 max_dev = MAX_RC_NUM * MAX_DEVICE_NUM;

	if (rc) {
		dev->rc_shadow[word_offset / 4] = wr_val;
	} else {
		for (i = 0; i < MAX_DEVICE_NUM; i++) {
			if (!dev->pcidev_table[i].bdf) {
				for (j = 0; j < max_dev; j++)
					if (!msm_pcie_dev_tbl[j].bdf) {
						msm_pcie_dev_tbl[j].bdf = bdf;
						break;
					}
				dev->pcidev_table[i].bdf = bdf;
				if ((!dev->bridge_found) && (i > 0))
					dev->bridge_found = true;
			}
			if (dev->pcidev_table[i].bdf == bdf) {
				dev->ep_shadow[i][word_offset / 4] = wr_val;
				break;
			}
		}
	}
}

static inline int msm_pcie_oper_conf(struct pci_bus *bus, u32 devfn, int oper,
				     int where, int size, u32 *val)
{
	uint32_t word_offset, byte_offset, mask;
	uint32_t rd_val, wr_val;
	struct msm_pcie_dev_t *dev;
	void __iomem *config_base;
	bool rc = false;
	u32 rc_idx;
	int rv = 0;
	u32 bdf = BDF_OFFSET(bus->number, devfn);
	int i;

	dev = PCIE_BUS_PRIV_DATA(bus);

	if (!dev) {
		pr_err("PCIe: No device found for this bus.\n");
		*val = ~0;
		rv = PCIBIOS_DEVICE_NOT_FOUND;
		goto out;
	}

	rc_idx = dev->rc_idx;
	rc = (bus->number == 0);

	spin_lock_irqsave(&dev->cfg_lock, dev->irqsave_flags);

	if (!dev->cfg_access) {
		PCIE_DBG3(dev,
			"Access denied for RC%d %d:0x%02x + 0x%04x[%d]\n",
			rc_idx, bus->number, devfn, where, size);
		*val = ~0;
		rv = PCIBIOS_DEVICE_NOT_FOUND;
		goto unlock;
	}

	if (rc && (devfn != 0)) {
		PCIE_DBG3(dev, "RC%d invalid %s - bus %d devfn %d\n", rc_idx,
			 (oper == RD) ? "rd" : "wr", bus->number, devfn);
		*val = ~0;
		rv = PCIBIOS_DEVICE_NOT_FOUND;
		goto unlock;
	}

	if (dev->link_status != MSM_PCIE_LINK_ENABLED) {
		PCIE_DBG3(dev,
			"Access to RC%d %d:0x%02x + 0x%04x[%d] is denied because link is down\n",
			rc_idx, bus->number, devfn, where, size);
		*val = ~0;
		rv = PCIBIOS_DEVICE_NOT_FOUND;
		goto unlock;
	}

	/* check if the link is up for endpoint */
	if (!rc && !msm_pcie_is_link_up(dev)) {
		PCIE_ERR(dev,
			"PCIe: RC%d %s fail, link down - bus %d devfn %d\n",
				rc_idx, (oper == RD) ? "rd" : "wr",
				bus->number, devfn);
			*val = ~0;
			rv = PCIBIOS_DEVICE_NOT_FOUND;
			goto unlock;
	}

	if (!rc && !dev->enumerated)
		msm_pcie_cfg_bdf(dev, bus->number, devfn);

	word_offset = where & ~0x3;
	byte_offset = where & 0x3;
	mask = ((u32)~0 >> (8 * (4 - size))) << (8 * byte_offset);

	if (rc || !dev->enumerated) {
		config_base = rc ? dev->dm_core : dev->conf;
	} else {
		for (i = 0; i < MAX_DEVICE_NUM; i++) {
			if (dev->pcidev_table[i].bdf == bdf) {
				config_base = dev->pcidev_table[i].conf_base;
				break;
			}
		}
		if (i == MAX_DEVICE_NUM) {
			*val = ~0;
			rv = PCIBIOS_DEVICE_NOT_FOUND;
			goto unlock;
		}
	}

	rd_val = readl_relaxed(config_base + word_offset);

	if (oper == RD) {
		*val = ((rd_val & mask) >> (8 * byte_offset));
		PCIE_DBG3(dev,
			"RC%d %d:0x%02x + 0x%04x[%d] -> 0x%08x; rd 0x%08x\n",
			rc_idx, bus->number, devfn, where, size, *val, rd_val);
	} else {
		wr_val = (rd_val & ~mask) |
				((*val << (8 * byte_offset)) & mask);

		if ((bus->number == 0) && (where == 0x3c))
			wr_val = wr_val | (3 << 16);

		writel_relaxed(wr_val, config_base + word_offset);
		wmb(); /* ensure config data is written to hardware register */

		if (dev->shadow_en) {
			if (rd_val == PCIE_LINK_DOWN &&
				(readl_relaxed(config_base) == PCIE_LINK_DOWN))
				PCIE_ERR(dev,
					"Read of RC%d %d:0x%02x + 0x%04x[%d] is all FFs\n",
					rc_idx, bus->number, devfn,
					where, size);
			else
				msm_pcie_save_shadow(dev, word_offset, wr_val,
					bdf, rc);
		}

		PCIE_DBG3(dev,
			"RC%d %d:0x%02x + 0x%04x[%d] <- 0x%08x; rd 0x%08x val 0x%08x\n",
			rc_idx, bus->number, devfn, where, size,
			wr_val, rd_val, *val);
	}

unlock:
	spin_unlock_irqrestore(&dev->cfg_lock, dev->irqsave_flags);
out:
	return rv;
}

static int msm_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where,
			    int size, u32 *val)
{
	int ret = msm_pcie_oper_conf(bus, devfn, RD, where, size, val);

	if ((bus->number == 0) && (where == PCI_CLASS_REVISION)) {
		*val = (*val & 0xff) | (PCI_CLASS_BRIDGE_PCI << 16);
		PCIE_GEN_DBG("change class for RC:0x%x\n", *val);
	}

	return ret;
}

static int msm_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
			    int where, int size, u32 val)
{
	return msm_pcie_oper_conf(bus, devfn, WR, where, size, &val);
}

static struct pci_ops msm_pcie_ops = {
	.read = msm_pcie_rd_conf,
	.write = msm_pcie_wr_conf,
};

static int msm_pcie_gpio_init(struct msm_pcie_dev_t *dev)
{
	int rc = 0, i;
	struct msm_pcie_gpio_info_t *info;

	PCIE_DBG(dev, "RC%d\n", dev->rc_idx);

	for (i = 0; i < dev->gpio_n; i++) {
		info = &dev->gpio[i];

		if (!info->num)
			continue;

		rc = gpio_request(info->num, info->name);
		if (rc) {
			PCIE_ERR(dev, "PCIe: RC%d can't get gpio %s; %d\n",
				dev->rc_idx, info->name, rc);
			break;
		}

		if (info->out)
			rc = gpio_direction_output(info->num, info->init);
		else
			rc = gpio_direction_input(info->num);
		if (rc) {
			PCIE_ERR(dev,
				"PCIe: RC%d can't set direction for GPIO %s:%d\n",
				dev->rc_idx, info->name, rc);
			gpio_free(info->num);
			break;
		}
	}

	if (rc)
		while (i--)
			gpio_free(dev->gpio[i].num);

	return rc;
}

static void msm_pcie_gpio_deinit(struct msm_pcie_dev_t *dev)
{
	int i;

	PCIE_DBG(dev, "RC%d\n", dev->rc_idx);

	for (i = 0; i < dev->gpio_n; i++)
		gpio_free(dev->gpio[i].num);
}

static int msm_pcie_vreg_init(struct msm_pcie_dev_t *dev)
{
	int i, rc = 0;
	struct regulator *vreg;
	struct msm_pcie_vreg_info_t *info;

	PCIE_DBG(dev, "RC%d: entry\n", dev->rc_idx);

	for (i = 0; i < MSM_PCIE_MAX_VREG; i++) {
		info = &dev->vreg[i];
		vreg = info->hdl;

		if (!vreg)
			continue;

		PCIE_DBG2(dev, "RC%d Vreg %s is being enabled\n",
			dev->rc_idx, info->name);
		if (info->max_v) {
			rc = regulator_set_voltage(vreg,
						   info->min_v, info->max_v);
			if (rc) {
				PCIE_ERR(dev,
					"PCIe: RC%d can't set voltage for %s: %d\n",
					dev->rc_idx, info->name, rc);
				break;
			}
		}

		if (info->opt_mode) {
			rc = regulator_set_load(vreg, info->opt_mode);
			if (rc < 0) {
				PCIE_ERR(dev,
					"PCIe: RC%d can't set mode for %s: %d\n",
					dev->rc_idx, info->name, rc);
				break;
			}
		}

		rc = regulator_enable(vreg);
		if (rc) {
			PCIE_ERR(dev,
				"PCIe: RC%d can't enable regulator %s: %d\n",
				dev->rc_idx, info->name, rc);
			break;
		}
	}

	if (rc)
		while (i--) {
			struct regulator *hdl = dev->vreg[i].hdl;

			if (hdl) {
				regulator_disable(hdl);
				if (!strcmp(dev->vreg[i].name, "vreg-cx")) {
					PCIE_DBG(dev,
						"RC%d: Removing %s vote.\n",
						dev->rc_idx,
						dev->vreg[i].name);
					regulator_set_voltage(hdl,
						RPMH_REGULATOR_LEVEL_OFF,
						RPMH_REGULATOR_LEVEL_MAX);
				}
			}

		}

	PCIE_DBG(dev, "RC%d: exit\n", dev->rc_idx);

	return rc;
}

static void msm_pcie_vreg_deinit(struct msm_pcie_dev_t *dev)
{
	int i;

	PCIE_DBG(dev, "RC%d: entry\n", dev->rc_idx);

	for (i = MSM_PCIE_MAX_VREG - 1; i >= 0; i--) {
		if (dev->vreg[i].hdl) {
			PCIE_DBG(dev, "Vreg %s is being disabled\n",
				dev->vreg[i].name);
			regulator_disable(dev->vreg[i].hdl);

			if (!strcmp(dev->vreg[i].name, "vreg-cx")) {
				PCIE_DBG(dev,
					"RC%d: Removing %s vote.\n",
					dev->rc_idx,
					dev->vreg[i].name);
				regulator_set_voltage(dev->vreg[i].hdl,
					RPMH_REGULATOR_LEVEL_OFF,
					RPMH_REGULATOR_LEVEL_MAX);
			}
		}
	}

	PCIE_DBG(dev, "RC%d: exit\n", dev->rc_idx);
}

static int msm_pcie_clk_init(struct msm_pcie_dev_t *dev)
{
	int i, rc = 0;
	struct msm_pcie_clk_info_t *info;
	struct msm_pcie_reset_info_t *reset_info;

	PCIE_DBG(dev, "RC%d: entry\n", dev->rc_idx);

	rc = regulator_enable(dev->gdsc);

	if (rc) {
		PCIE_ERR(dev, "PCIe: fail to enable GDSC for RC%d (%s)\n",
			dev->rc_idx, dev->pdev->name);
		return rc;
	}

	if (dev->gdsc_smmu) {
		rc = regulator_enable(dev->gdsc_smmu);

		if (rc) {
			PCIE_ERR(dev,
				"PCIe: fail to enable SMMU GDSC for RC%d (%s)\n",
				dev->rc_idx, dev->pdev->name);
			return rc;
		}
	}

	PCIE_DBG(dev, "PCIe: requesting bus vote for RC%d\n", dev->rc_idx);
	if (dev->bus_client) {
		rc = msm_bus_scale_client_update_request(dev->bus_client, 1);
		if (rc) {
			PCIE_ERR(dev,
				"PCIe: fail to set bus bandwidth for RC%d:%d.\n",
				dev->rc_idx, rc);
			return rc;
		}

		PCIE_DBG2(dev,
			"PCIe: set bus bandwidth for RC%d.\n",
			dev->rc_idx);
	}

	for (i = 0; i < MSM_PCIE_MAX_CLK; i++) {
		info = &dev->clk[i];

		if (!info->hdl)
			continue;

		if (info->config_mem)
			msm_pcie_config_clock_mem(dev, info);

		if (info->freq) {
			rc = clk_set_rate(info->hdl, info->freq);
			if (rc) {
				PCIE_ERR(dev,
					"PCIe: RC%d can't set rate for clk %s: %d.\n",
					dev->rc_idx, info->name, rc);
				break;
			}

			PCIE_DBG2(dev,
				"PCIe: RC%d set rate for clk %s.\n",
				dev->rc_idx, info->name);
		}

		rc = clk_prepare_enable(info->hdl);

		if (rc)
			PCIE_ERR(dev, "PCIe: RC%d failed to enable clk %s\n",
				dev->rc_idx, info->name);
		else
			PCIE_DBG2(dev, "enable clk %s for RC%d.\n",
				info->name, dev->rc_idx);
	}

	if (rc) {
		PCIE_DBG(dev, "RC%d disable clocks for error handling.\n",
			dev->rc_idx);
		while (i--) {
			struct clk *hdl = dev->clk[i].hdl;

			if (hdl)
				clk_disable_unprepare(hdl);
		}

		if (dev->gdsc_smmu)
			regulator_disable(dev->gdsc_smmu);

		regulator_disable(dev->gdsc);
	}

	for (i = 0; i < MSM_PCIE_MAX_RESET; i++) {
		reset_info = &dev->reset[i];
		if (reset_info->hdl) {
			rc = reset_control_assert(reset_info->hdl);
			if (rc)
				PCIE_ERR(dev,
					"PCIe: RC%d failed to assert reset for %s.\n",
					dev->rc_idx, reset_info->name);
			else
				PCIE_DBG2(dev,
					"PCIe: RC%d successfully asserted reset for %s.\n",
					dev->rc_idx, reset_info->name);

			/* add a 1ms delay to ensure the reset is asserted */
			usleep_range(1000, 1005);

			rc = reset_control_deassert(reset_info->hdl);
			if (rc)
				PCIE_ERR(dev,
					"PCIe: RC%d failed to deassert reset for %s.\n",
					dev->rc_idx, reset_info->name);
			else
				PCIE_DBG2(dev,
					"PCIe: RC%d successfully deasserted reset for %s.\n",
					dev->rc_idx, reset_info->name);
		}
	}

	PCIE_DBG(dev, "RC%d: exit\n", dev->rc_idx);

	return rc;
}

static void msm_pcie_clk_deinit(struct msm_pcie_dev_t *dev)
{
	int i;
	int rc;

	PCIE_DBG(dev, "RC%d: entry\n", dev->rc_idx);

	for (i = 0; i < MSM_PCIE_MAX_CLK; i++)
		if (dev->clk[i].hdl)
			clk_disable_unprepare(dev->clk[i].hdl);

	if (dev->bus_client) {
		PCIE_DBG(dev, "PCIe: removing bus vote for RC%d\n",
			dev->rc_idx);

		rc = msm_bus_scale_client_update_request(dev->bus_client, 0);
		if (rc)
			PCIE_ERR(dev,
				"PCIe: fail to relinquish bus bandwidth for RC%d:%d.\n",
				dev->rc_idx, rc);
		else
			PCIE_DBG(dev,
				"PCIe: relinquish bus bandwidth for RC%d.\n",
				dev->rc_idx);
	}

	if (dev->gdsc_smmu)
		regulator_disable(dev->gdsc_smmu);

	regulator_disable(dev->gdsc);

	PCIE_DBG(dev, "RC%d: exit\n", dev->rc_idx);
}

static int msm_pcie_pipe_clk_init(struct msm_pcie_dev_t *dev)
{
	int i, rc = 0;
	struct msm_pcie_clk_info_t *info;
	struct msm_pcie_reset_info_t *pipe_reset_info;

	PCIE_DBG(dev, "RC%d: entry\n", dev->rc_idx);

	for (i = 0; i < MSM_PCIE_MAX_PIPE_CLK; i++) {
		info = &dev->pipeclk[i];

		if (!info->hdl)
			continue;


		if (info->config_mem)
			msm_pcie_config_clock_mem(dev, info);

		if (info->freq) {
			rc = clk_set_rate(info->hdl, info->freq);
			if (rc) {
				PCIE_ERR(dev,
					"PCIe: RC%d can't set rate for clk %s: %d.\n",
					dev->rc_idx, info->name, rc);
				break;
			}

			PCIE_DBG2(dev,
				"PCIe: RC%d set rate for clk %s: %d.\n",
				dev->rc_idx, info->name, rc);
		}

		rc = clk_prepare_enable(info->hdl);

		if (rc)
			PCIE_ERR(dev, "PCIe: RC%d failed to enable clk %s.\n",
				dev->rc_idx, info->name);
		else
			PCIE_DBG2(dev, "RC%d enabled pipe clk %s.\n",
				dev->rc_idx, info->name);
	}

	if (rc) {
		PCIE_DBG(dev, "RC%d disable pipe clocks for error handling.\n",
			dev->rc_idx);
		while (i--)
			if (dev->pipeclk[i].hdl)
				clk_disable_unprepare(dev->pipeclk[i].hdl);
	}

	for (i = 0; i < MSM_PCIE_MAX_PIPE_RESET; i++) {
		pipe_reset_info = &dev->pipe_reset[i];
		if (pipe_reset_info->hdl) {
			rc = reset_control_assert(pipe_reset_info->hdl);
			if (rc)
				PCIE_ERR(dev,
					"PCIe: RC%d failed to assert pipe reset for %s.\n",
					dev->rc_idx, pipe_reset_info->name);
			else
				PCIE_DBG2(dev,
					"PCIe: RC%d successfully asserted pipe reset for %s.\n",
					dev->rc_idx, pipe_reset_info->name);

			/* add a 1ms delay to ensure the reset is asserted */
			usleep_range(1000, 1005);

			rc = reset_control_deassert(
					pipe_reset_info->hdl);
			if (rc)
				PCIE_ERR(dev,
					"PCIe: RC%d failed to deassert pipe reset for %s.\n",
					dev->rc_idx, pipe_reset_info->name);
			else
				PCIE_DBG2(dev,
					"PCIe: RC%d successfully deasserted pipe reset for %s.\n",
					dev->rc_idx, pipe_reset_info->name);
		}
	}

	PCIE_DBG(dev, "RC%d: exit\n", dev->rc_idx);

	return rc;
}

static void msm_pcie_pipe_clk_deinit(struct msm_pcie_dev_t *dev)
{
	int i;

	PCIE_DBG(dev, "RC%d: entry\n", dev->rc_idx);

	for (i = 0; i < MSM_PCIE_MAX_PIPE_CLK; i++)
		if (dev->pipeclk[i].hdl)
			clk_disable_unprepare(
				dev->pipeclk[i].hdl);

	PCIE_DBG(dev, "RC%d: exit\n", dev->rc_idx);
}

static void msm_pcie_iatu_config_all_ep(struct msm_pcie_dev_t *dev)
{
	int i;
	u8 type;
	struct msm_pcie_device_info *dev_table = dev->pcidev_table;

	for (i = 0; i < MAX_DEVICE_NUM; i++) {
		if (!dev_table[i].bdf)
			break;

		type = dev_table[i].bdf >> 24 == 0x1 ?
			PCIE20_CTRL1_TYPE_CFG0 : PCIE20_CTRL1_TYPE_CFG1;

		msm_pcie_iatu_config(dev, i, type, dev_table[i].phy_address,
			dev_table[i].phy_address + SZ_4K - 1,
			dev_table[i].bdf);
	}
}

static void msm_pcie_config_controller(struct msm_pcie_dev_t *dev)
{
	PCIE_DBG(dev, "RC%d\n", dev->rc_idx);

	/*
	 * program and enable address translation region 0 (device config
	 * address space); region type config;
	 * axi config address range to device config address range
	 */
	if (dev->enumerated) {
		msm_pcie_iatu_config_all_ep(dev);
	} else {
		dev->current_bdf = 0; /* to force IATU re-config */
		msm_pcie_cfg_bdf(dev, 1, 0);
	}

	/* configure N_FTS */
	PCIE_DBG2(dev, "Original PCIE20_ACK_F_ASPM_CTRL_REG:0x%x\n",
		readl_relaxed(dev->dm_core + PCIE20_ACK_F_ASPM_CTRL_REG));
	if (!dev->n_fts)
		msm_pcie_write_mask(dev->dm_core + PCIE20_ACK_F_ASPM_CTRL_REG,
					0, BIT(15));
	else
		msm_pcie_write_mask(dev->dm_core + PCIE20_ACK_F_ASPM_CTRL_REG,
					PCIE20_ACK_N_FTS,
					dev->n_fts << 8);

	if (dev->shadow_en)
		dev->rc_shadow[PCIE20_ACK_F_ASPM_CTRL_REG / 4] =
			readl_relaxed(dev->dm_core +
			PCIE20_ACK_F_ASPM_CTRL_REG);

	PCIE_DBG2(dev, "Updated PCIE20_ACK_F_ASPM_CTRL_REG:0x%x\n",
		readl_relaxed(dev->dm_core + PCIE20_ACK_F_ASPM_CTRL_REG));

	/* configure AUX clock frequency register for PCIe core */
	if (dev->use_19p2mhz_aux_clk)
		msm_pcie_write_reg(dev->dm_core, PCIE20_AUX_CLK_FREQ_REG, 0x14);
	else
		msm_pcie_write_reg(dev->dm_core, PCIE20_AUX_CLK_FREQ_REG, 0x01);

	/* configure the completion timeout value for PCIe core */
	if (dev->cpl_timeout && dev->bridge_found)
		msm_pcie_write_reg_field(dev->dm_core,
					PCIE20_DEVICE_CONTROL2_STATUS2,
					0xf, dev->cpl_timeout);

	/* Enable AER on RC */
	if (dev->aer_enable) {
		msm_pcie_write_mask(dev->dm_core + PCIE20_BRIDGE_CTRL, 0,
						BIT(16)|BIT(17));
		msm_pcie_write_mask(dev->dm_core +  PCIE20_CAP_DEVCTRLSTATUS, 0,
						BIT(3)|BIT(2)|BIT(1)|BIT(0));

		PCIE_DBG(dev, "RC's PCIE20_CAP_DEVCTRLSTATUS:0x%x\n",
			readl_relaxed(dev->dm_core + PCIE20_CAP_DEVCTRLSTATUS));
	}
}

static void msm_pcie_config_msi_controller(struct msm_pcie_dev_t *dev)
{
	int i;

	PCIE_DBG(dev, "RC%d\n", dev->rc_idx);

	/* program MSI controller and enable all interrupts */
	writel_relaxed(MSM_PCIE_MSI_PHY, dev->dm_core + PCIE20_MSI_CTRL_ADDR);
	writel_relaxed(0, dev->dm_core + PCIE20_MSI_CTRL_UPPER_ADDR);

	for (i = 0; i < PCIE20_MSI_CTRL_MAX; i++)
		writel_relaxed(~0, dev->dm_core +
			       PCIE20_MSI_CTRL_INTR_EN + (i * 12));

	/* ensure that hardware is configured before proceeding */
	wmb();
}

static int msm_pcie_get_resources(struct msm_pcie_dev_t *dev,
					struct platform_device *pdev)
{
	int i, len, cnt, ret = 0, size = 0;
	struct msm_pcie_vreg_info_t *vreg_info;
	struct msm_pcie_gpio_info_t *gpio_info;
	struct msm_pcie_clk_info_t  *clk_info;
	struct resource *res;
	struct msm_pcie_res_info_t *res_info;
	struct msm_pcie_irq_info_t *irq_info;
	struct msm_pcie_irq_info_t *msi_info;
	struct msm_pcie_reset_info_t *reset_info;
	struct msm_pcie_reset_info_t *pipe_reset_info;
	char prop_name[MAX_PROP_SIZE];
	const __be32 *prop;
	u32 *clkfreq = NULL;

	PCIE_DBG(dev, "RC%d: entry\n", dev->rc_idx);

	cnt = of_property_count_elems_of_size((&pdev->dev)->of_node,
			"max-clock-frequency-hz", sizeof(u32));
	if (cnt > 0) {
		clkfreq = kzalloc((MSM_PCIE_MAX_CLK + MSM_PCIE_MAX_PIPE_CLK) *
					sizeof(*clkfreq), GFP_KERNEL);
		if (!clkfreq) {
			PCIE_ERR(dev, "PCIe: memory alloc failed for RC%d\n",
					dev->rc_idx);
			return -ENOMEM;
		}
		ret = of_property_read_u32_array(
			(&pdev->dev)->of_node,
			"max-clock-frequency-hz", clkfreq, cnt);
		if (ret) {
			PCIE_ERR(dev,
				"PCIe: invalid max-clock-frequency-hz property for RC%d:%d\n",
				dev->rc_idx, ret);
			goto out;
		}
	}

	for (i = 0; i < MSM_PCIE_MAX_VREG; i++) {
		vreg_info = &dev->vreg[i];
		vreg_info->hdl =
				devm_regulator_get(&pdev->dev, vreg_info->name);

		if (PTR_ERR(vreg_info->hdl) == -EPROBE_DEFER) {
			PCIE_DBG(dev, "EPROBE_DEFER for VReg:%s\n",
				vreg_info->name);
			ret = PTR_ERR(vreg_info->hdl);
			goto out;
		}

		if (IS_ERR(vreg_info->hdl)) {
			if (vreg_info->required) {
				PCIE_DBG(dev, "Vreg %s doesn't exist\n",
					vreg_info->name);
				ret = PTR_ERR(vreg_info->hdl);
				goto out;
			} else {
				PCIE_DBG(dev,
					"Optional Vreg %s doesn't exist\n",
					vreg_info->name);
				vreg_info->hdl = NULL;
			}
		} else {
			dev->vreg_n++;
			snprintf(prop_name, MAX_PROP_SIZE,
				"qcom,%s-voltage-level", vreg_info->name);
			prop = of_get_property((&pdev->dev)->of_node,
						prop_name, &len);
			if (!prop || (len != (3 * sizeof(__be32)))) {
				PCIE_DBG(dev, "%s %s property\n",
					prop ? "invalid format" :
					"no", prop_name);
			} else {
				vreg_info->max_v = be32_to_cpup(&prop[0]);
				vreg_info->min_v = be32_to_cpup(&prop[1]);
				vreg_info->opt_mode =
					be32_to_cpup(&prop[2]);
			}
		}
	}

	dev->gdsc = devm_regulator_get(&pdev->dev, "gdsc-vdd");

	if (IS_ERR(dev->gdsc)) {
		PCIE_ERR(dev, "PCIe: RC%d Failed to get %s GDSC:%ld\n",
			dev->rc_idx, dev->pdev->name, PTR_ERR(dev->gdsc));
		if (PTR_ERR(dev->gdsc) == -EPROBE_DEFER)
			PCIE_DBG(dev, "PCIe: EPROBE_DEFER for %s GDSC\n",
					dev->pdev->name);
		ret = PTR_ERR(dev->gdsc);
		goto out;
	}

	dev->gdsc_smmu = devm_regulator_get(&pdev->dev, "gdsc-smmu");

	if (IS_ERR(dev->gdsc_smmu)) {
		PCIE_DBG(dev, "PCIe: RC%d SMMU GDSC does not exist",
			dev->rc_idx);
		dev->gdsc_smmu = NULL;
	}

	dev->gpio_n = 0;
	for (i = 0; i < MSM_PCIE_MAX_GPIO; i++) {
		gpio_info = &dev->gpio[i];
		ret = of_get_named_gpio((&pdev->dev)->of_node,
					gpio_info->name, 0);
		if (ret >= 0) {
			gpio_info->num = ret;
			dev->gpio_n++;
			PCIE_DBG(dev, "GPIO num for %s is %d\n",
				gpio_info->name, gpio_info->num);
		} else {
			if (gpio_info->required) {
				PCIE_ERR(dev,
					"Could not get required GPIO %s\n",
					gpio_info->name);
				goto out;
			} else {
				PCIE_DBG(dev,
					"Could not get optional GPIO %s\n",
					gpio_info->name);
			}
		}
		ret = 0;
	}

	of_get_property(pdev->dev.of_node, "qcom,phy-sequence", &size);
	if (size) {
		dev->phy_sequence = (struct msm_pcie_phy_info_t *)
			devm_kzalloc(&pdev->dev, size, GFP_KERNEL);

		if (dev->phy_sequence) {
			dev->phy_len =
				size / sizeof(*dev->phy_sequence);

			of_property_read_u32_array(pdev->dev.of_node,
				"qcom,phy-sequence",
				(unsigned int *)dev->phy_sequence,
				size / sizeof(dev->phy_sequence->offset));
		} else {
			PCIE_ERR(dev,
				"RC%d: Could not allocate memory for phy init sequence.\n",
				dev->rc_idx);
			ret = -ENOMEM;
			goto out;
		}
	} else {
		PCIE_DBG(dev, "RC%d: phy sequence is not present in DT\n",
			dev->rc_idx);
	}

	for (i = 0; i < MSM_PCIE_MAX_CLK; i++) {
		clk_info = &dev->clk[i];

		clk_info->hdl = devm_clk_get(&pdev->dev, clk_info->name);

		if (IS_ERR(clk_info->hdl)) {
			if (clk_info->required) {
				PCIE_DBG(dev, "Clock %s isn't available:%ld\n",
				clk_info->name, PTR_ERR(clk_info->hdl));
				ret = PTR_ERR(clk_info->hdl);
				goto out;
			} else {
				PCIE_DBG(dev, "Ignoring Clock %s\n",
					clk_info->name);
				clk_info->hdl = NULL;
			}
		} else {
			if (clkfreq != NULL) {
				clk_info->freq = clkfreq[i +
					MSM_PCIE_MAX_PIPE_CLK];
				PCIE_DBG(dev, "Freq of Clock %s is:%d\n",
					clk_info->name, clk_info->freq);
			}
		}
	}

	for (i = 0; i < MSM_PCIE_MAX_PIPE_CLK; i++) {
		clk_info = &dev->pipeclk[i];

		clk_info->hdl = devm_clk_get(&pdev->dev, clk_info->name);

		if (IS_ERR(clk_info->hdl)) {
			if (clk_info->required) {
				PCIE_DBG(dev, "Clock %s isn't available:%ld\n",
				clk_info->name, PTR_ERR(clk_info->hdl));
				ret = PTR_ERR(clk_info->hdl);
				goto out;
			} else {
				PCIE_DBG(dev, "Ignoring Clock %s\n",
					clk_info->name);
				clk_info->hdl = NULL;
			}
		} else {
			if (clkfreq != NULL) {
				clk_info->freq = clkfreq[i];
				PCIE_DBG(dev, "Freq of Clock %s is:%d\n",
					clk_info->name, clk_info->freq);
			}
		}
	}

	for (i = 0; i < MSM_PCIE_MAX_RESET; i++) {
		reset_info = &dev->reset[i];

		reset_info->hdl = devm_reset_control_get(&pdev->dev,
						reset_info->name);

		if (IS_ERR(reset_info->hdl)) {
			if (reset_info->required) {
				PCIE_DBG(dev,
					"Reset %s isn't available:%ld\n",
					reset_info->name,
					PTR_ERR(reset_info->hdl));

				ret = PTR_ERR(reset_info->hdl);
				reset_info->hdl = NULL;
				goto out;
			} else {
				PCIE_DBG(dev, "Ignoring Reset %s\n",
					reset_info->name);
				reset_info->hdl = NULL;
			}
		}
	}

	for (i = 0; i < MSM_PCIE_MAX_PIPE_RESET; i++) {
		pipe_reset_info = &dev->pipe_reset[i];

		pipe_reset_info->hdl = devm_reset_control_get(&pdev->dev,
						pipe_reset_info->name);

		if (IS_ERR(pipe_reset_info->hdl)) {
			if (pipe_reset_info->required) {
				PCIE_DBG(dev,
					"Pipe Reset %s isn't available:%ld\n",
					pipe_reset_info->name,
					PTR_ERR(pipe_reset_info->hdl));

				ret = PTR_ERR(pipe_reset_info->hdl);
				pipe_reset_info->hdl = NULL;
				goto out;
			} else {
				PCIE_DBG(dev, "Ignoring Pipe Reset %s\n",
					pipe_reset_info->name);
				pipe_reset_info->hdl = NULL;
			}
		}
	}

	dev->bus_scale_table = msm_bus_cl_get_pdata(pdev);
	if (!dev->bus_scale_table) {
		PCIE_DBG(dev, "PCIe: No bus scale table for RC%d (%s)\n",
			dev->rc_idx, dev->pdev->name);
		dev->bus_client = 0;
	} else {
		dev->bus_client =
			msm_bus_scale_register_client(dev->bus_scale_table);
		if (!dev->bus_client) {
			PCIE_ERR(dev,
				"PCIe: Failed to register bus client for RC%d (%s)\n",
				dev->rc_idx, dev->pdev->name);
			msm_bus_cl_clear_pdata(dev->bus_scale_table);
			ret = -ENODEV;
			goto out;
		}
	}

	for (i = 0; i < MSM_PCIE_MAX_RES; i++) {
		res_info = &dev->res[i];

		res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
							   res_info->name);

		if (!res) {
			PCIE_ERR(dev, "PCIe: RC%d can't get %s resource.\n",
				dev->rc_idx, res_info->name);
		} else {
			PCIE_DBG(dev, "start addr for %s is %pa.\n",
				res_info->name,	&res->start);

			res_info->base = devm_ioremap(&pdev->dev,
						res->start, resource_size(res));
			if (!res_info->base) {
				PCIE_ERR(dev, "PCIe: RC%d can't remap %s.\n",
					dev->rc_idx, res_info->name);
				ret = -ENOMEM;
				goto out;
			} else {
				res_info->resource = res;
			}
		}
	}

	for (i = 0; i < MSM_PCIE_MAX_IRQ; i++) {
		irq_info = &dev->irq[i];

		res = platform_get_resource_byname(pdev, IORESOURCE_IRQ,
							   irq_info->name);

		if (!res) {
			PCIE_DBG(dev, "PCIe: RC%d can't find IRQ # for %s.\n",
				dev->rc_idx, irq_info->name);
		} else {
			irq_info->num = res->start;
			PCIE_DBG(dev, "IRQ # for %s is %d.\n", irq_info->name,
					irq_info->num);
		}
	}

	for (i = 0; i < MSM_PCIE_MAX_MSI; i++) {
		msi_info = &dev->msi[i];

		res = platform_get_resource_byname(pdev, IORESOURCE_IRQ,
							   msi_info->name);

		if (!res) {
			PCIE_DBG(dev, "PCIe: RC%d can't find IRQ # for %s.\n",
				dev->rc_idx, msi_info->name);
		} else {
			msi_info->num = res->start;
			PCIE_DBG(dev, "IRQ # for %s is %d.\n", msi_info->name,
					msi_info->num);
		}
	}

	/* All allocations succeeded */

	if (dev->gpio[MSM_PCIE_GPIO_WAKE].num)
		dev->wake_n = gpio_to_irq(dev->gpio[MSM_PCIE_GPIO_WAKE].num);
	else
		dev->wake_n = 0;

	dev->parf = dev->res[MSM_PCIE_RES_PARF].base;
	dev->phy = dev->res[MSM_PCIE_RES_PHY].base;
	dev->elbi = dev->res[MSM_PCIE_RES_ELBI].base;
	dev->iatu = dev->res[MSM_PCIE_RES_IATU].base;
	dev->dm_core = dev->res[MSM_PCIE_RES_DM_CORE].base;
	dev->conf = dev->res[MSM_PCIE_RES_CONF].base;
	dev->bars = dev->res[MSM_PCIE_RES_BARS].base;
	dev->tcsr = dev->res[MSM_PCIE_RES_TCSR].base;
	dev->dev_mem_res = dev->res[MSM_PCIE_RES_BARS].resource;
	dev->dev_io_res = dev->res[MSM_PCIE_RES_IO].resource;
	dev->dev_io_res->flags = IORESOURCE_IO;

out:
	kfree(clkfreq);

	PCIE_DBG(dev, "RC%d: exit\n", dev->rc_idx);

	return ret;
}

static void msm_pcie_release_resources(struct msm_pcie_dev_t *dev)
{
	dev->parf = NULL;
	dev->elbi = NULL;
	dev->iatu = NULL;
	dev->dm_core = NULL;
	dev->conf = NULL;
	dev->bars = NULL;
	dev->tcsr = NULL;
	dev->dev_mem_res = NULL;
	dev->dev_io_res = NULL;
}

static void msm_pcie_setup_gen3(struct msm_pcie_dev_t *dev)
{
	PCIE_DBG(dev, "PCIe: RC%d: Setting up Gen3\n", dev->rc_idx);

	msm_pcie_write_reg_field(dev->dm_core,
		PCIE_GEN3_L0_DRVR_CTRL0, 0x1ff00, BIT(0));

	msm_pcie_write_reg(dev->dm_core,
		PCIE_GEN3_L0_BIST_ERR_CNT2_STATUS,
		(0x05 << 14) | (0x05 << 10) | (0x0d <<  5));

	msm_pcie_write_mask(dev->dm_core +
		PCIE_GEN3_L0_BIST_ERR_CNT1_STATUS, BIT(4), 0);

	msm_pcie_write_mask(dev->dm_core +
		PCIE_GEN3_L0_RESET_GEN, BIT(0), 0);

	/* configure PCIe preset */
	msm_pcie_write_reg(dev->dm_core,
		PCIE_GEN3_L0_DEBUG_BUS_STATUS4, 1);
	msm_pcie_write_reg(dev->dm_core,
		PCIE_GEN3_COM_INTEGLOOP_GAIN1_MODE0, 0x77777777);
	msm_pcie_write_reg(dev->dm_core,
		PCIE_GEN3_L0_DEBUG_BUS_STATUS4, 1);

	msm_pcie_write_reg_field(dev->dm_core,
		PCIE20_CAP + PCI_EXP_LNKCTL2,
		PCI_EXP_LNKCAP_SLS, GEN3_SPEED);
}

static int msm_pcie_enable(struct msm_pcie_dev_t *dev, u32 options)
{
	int ret = 0;
	uint32_t val;
	long int retries = 0;
	int link_check_count = 0;
	unsigned long ep_up_timeout = 0;
	u32 link_check_max_count;

	PCIE_DBG(dev, "RC%d: entry\n", dev->rc_idx);

	mutex_lock(&dev->setup_lock);

	if (dev->link_status == MSM_PCIE_LINK_ENABLED) {
		PCIE_ERR(dev, "PCIe: the link of RC%d is already enabled\n",
			dev->rc_idx);
		goto out;
	}

	/* assert PCIe reset link to keep EP in reset */

	PCIE_INFO(dev, "PCIe: Assert the reset of endpoint of RC%d.\n",
		dev->rc_idx);
	gpio_set_value(dev->gpio[MSM_PCIE_GPIO_PERST].num,
				dev->gpio[MSM_PCIE_GPIO_PERST].on);
	usleep_range(PERST_PROPAGATION_DELAY_US_MIN,
				 PERST_PROPAGATION_DELAY_US_MAX);

	/* enable power */

	if (options & PM_VREG) {
		ret = msm_pcie_vreg_init(dev);
		if (ret)
			goto out;
	}

	/* enable clocks */
	if (options & PM_CLK) {
		ret = msm_pcie_clk_init(dev);
		/* ensure that changes propagated to the hardware */
		wmb();
		if (ret)
			goto clk_fail;
	}

	if (dev->scm_dev_id) {
		PCIE_DBG(dev, "RC%d: restoring sec config\n", dev->rc_idx);
		msm_pcie_restore_sec_config(dev);
	}

	/* configure PCIe to RC mode */
	msm_pcie_write_reg(dev->parf, PCIE20_PARF_DEVICE_TYPE, 0x4);

	/* enable l1 mode, clear bit 5 (REQ_NOT_ENTR_L1) */
	if (dev->l1_supported)
		msm_pcie_write_mask(dev->parf + PCIE20_PARF_PM_CTRL, BIT(5), 0);

	/* enable PCIe clocks and resets */
	msm_pcie_write_mask(dev->parf + PCIE20_PARF_PHY_CTRL, BIT(0), 0);

	/* change DBI base address */
	writel_relaxed(0, dev->parf + PCIE20_PARF_DBI_BASE_ADDR);

	writel_relaxed(0x365E, dev->parf + PCIE20_PARF_SYS_CTRL);

	msm_pcie_write_mask(dev->parf + PCIE20_PARF_MHI_CLOCK_RESET_CTRL,
				0, BIT(4));

	/* enable selected IRQ */
	if (dev->irq[MSM_PCIE_INT_GLOBAL_INT].num) {
		msm_pcie_write_reg(dev->parf, PCIE20_PARF_INT_ALL_MASK, 0);

		msm_pcie_write_mask(dev->parf + PCIE20_PARF_INT_ALL_MASK, 0,
					BIT(MSM_PCIE_INT_EVT_LINK_DOWN) |
					BIT(MSM_PCIE_INT_EVT_AER_LEGACY) |
					BIT(MSM_PCIE_INT_EVT_AER_ERR) |
					BIT(MSM_PCIE_INT_EVT_MSI_0) |
					BIT(MSM_PCIE_INT_EVT_MSI_1) |
					BIT(MSM_PCIE_INT_EVT_MSI_2) |
					BIT(MSM_PCIE_INT_EVT_MSI_3) |
					BIT(MSM_PCIE_INT_EVT_MSI_4) |
					BIT(MSM_PCIE_INT_EVT_MSI_5) |
					BIT(MSM_PCIE_INT_EVT_MSI_6) |
					BIT(MSM_PCIE_INT_EVT_MSI_7));

		PCIE_DBG(dev, "PCIe: RC%d: PCIE20_PARF_INT_ALL_MASK: 0x%x\n",
			dev->rc_idx,
			readl_relaxed(dev->parf + PCIE20_PARF_INT_ALL_MASK));
	}

	writel_relaxed(dev->slv_addr_space_size, dev->parf +
		PCIE20_PARF_SLV_ADDR_SPACE_SIZE);

	if (dev->use_msi) {
		PCIE_DBG(dev, "RC%d: enable WR halt.\n", dev->rc_idx);
		val = dev->wr_halt_size ? dev->wr_halt_size :
			readl_relaxed(dev->parf +
				PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT);

		msm_pcie_write_reg(dev->parf,
			PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT,
			BIT(31) | val);

		PCIE_DBG(dev,
			"RC%d: PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT: 0x%x.\n",
			dev->rc_idx,
			readl_relaxed(dev->parf +
				PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT));
	}

	/* init PCIe PHY */
	pcie_phy_init(dev);

	if (options & PM_PIPE_CLK) {
		usleep_range(PHY_STABILIZATION_DELAY_US_MIN,
					 PHY_STABILIZATION_DELAY_US_MAX);
		/* Enable the pipe clock */
		ret = msm_pcie_pipe_clk_init(dev);
		/* ensure that changes propagated to the hardware */
		wmb();
		if (ret)
			goto link_fail;
	}

	/* check capability for max link speed */
	if (!dev->max_link_speed) {
		val = readl_relaxed(dev->dm_core + PCIE20_CAP + PCI_EXP_LNKCAP);
		dev->max_link_speed = val & PCI_EXP_LNKCAP_SLS;
	}

	PCIE_DBG(dev, "RC%d: waiting for phy ready...\n", dev->rc_idx);

	do {
		if (pcie_phy_is_ready(dev))
			break;
		retries++;
		usleep_range(REFCLK_STABILIZATION_DELAY_US_MIN,
					 REFCLK_STABILIZATION_DELAY_US_MAX);
	} while (retries < PHY_READY_TIMEOUT_COUNT);

	PCIE_DBG(dev, "RC%d: number of PHY retries:%ld.\n",
		dev->rc_idx, retries);

	if (pcie_phy_is_ready(dev))
		PCIE_INFO(dev, "PCIe RC%d PHY is ready!\n", dev->rc_idx);
	else {
		PCIE_ERR(dev, "PCIe PHY RC%d failed to come up!\n",
			dev->rc_idx);
		ret = -ENODEV;
		pcie_phy_dump(dev);
		goto link_fail;
	}

	if (dev->ep_latency)
		usleep_range(dev->ep_latency * 1000, dev->ep_latency * 1000);

	if (dev->gpio[MSM_PCIE_GPIO_EP].num)
		gpio_set_value(dev->gpio[MSM_PCIE_GPIO_EP].num,
				dev->gpio[MSM_PCIE_GPIO_EP].on);

	/* de-assert PCIe reset link to bring EP out of reset */

	PCIE_INFO(dev, "PCIe: Release the reset of endpoint of RC%d.\n",
		dev->rc_idx);
	gpio_set_value(dev->gpio[MSM_PCIE_GPIO_PERST].num,
				1 - dev->gpio[MSM_PCIE_GPIO_PERST].on);
	usleep_range(dev->perst_delay_us_min, dev->perst_delay_us_max);

	ep_up_timeout = jiffies + usecs_to_jiffies(EP_UP_TIMEOUT_US);

	/* setup Gen3 specific configurations */
	if (dev->max_link_speed == GEN3_SPEED)
		msm_pcie_setup_gen3(dev);

	if (msm_pcie_force_gen1 & BIT(dev->rc_idx))
		msm_pcie_write_reg_field(dev->dm_core,
			PCIE20_CAP + PCI_EXP_LNKCTL2,
			PCI_EXP_LNKCAP_SLS, GEN1_SPEED);

	/* set max tlp read size */
	msm_pcie_write_reg_field(dev->dm_core, PCIE20_DEVICE_CONTROL_STATUS,
				0x7000, dev->tlp_rd_size);

	/* enable link training */
	msm_pcie_write_mask(dev->parf + PCIE20_PARF_LTSSM, 0, BIT(8));

	PCIE_DBG(dev, "%s", "check if link is up\n");

	if (msm_pcie_link_check_max_count & BIT(dev->rc_idx))
		link_check_max_count = msm_pcie_link_check_max_count >> 4;
	else
		link_check_max_count = LINK_UP_CHECK_MAX_COUNT;

	/* Wait for up to 100ms for the link to come up */
	do {
		usleep_range(LINK_UP_TIMEOUT_US_MIN, LINK_UP_TIMEOUT_US_MAX);
		val =  readl_relaxed(dev->elbi + PCIE20_ELBI_SYS_STTS);
		PCIE_DBG(dev, "PCIe RC%d: LTSSM_STATE:0x%x\n",
			dev->rc_idx, (val >> 12) & 0x3f);
	} while ((!(val & XMLH_LINK_UP) ||
		!msm_pcie_confirm_linkup(dev, false, false, NULL))
		&& (link_check_count++ < link_check_max_count));

	if ((val & XMLH_LINK_UP) &&
		msm_pcie_confirm_linkup(dev, false, false, NULL)) {
		PCIE_DBG(dev, "Link is up after %d checkings\n",
			link_check_count);
		PCIE_INFO(dev, "PCIe RC%d link initialized\n", dev->rc_idx);
	} else {
		PCIE_INFO(dev, "PCIe: Assert the reset of endpoint of RC%d.\n",
			dev->rc_idx);
		gpio_set_value(dev->gpio[MSM_PCIE_GPIO_PERST].num,
			dev->gpio[MSM_PCIE_GPIO_PERST].on);
		PCIE_ERR(dev, "PCIe RC%d link initialization failed\n",
			dev->rc_idx);
		ret = -1;
		goto link_fail;
	}

	dev->link_status = MSM_PCIE_LINK_ENABLED;
	dev->power_on = true;
	dev->suspending = false;
	dev->link_turned_on_counter++;

	if (dev->switch_latency) {
		PCIE_DBG(dev, "switch_latency: %dms\n",
			dev->switch_latency);
		if (dev->switch_latency <= SWITCH_DELAY_MAX)
			usleep_range(dev->switch_latency * 1000,
				dev->switch_latency * 1000);
		else
			msleep(dev->switch_latency);
	}

	msm_pcie_config_controller(dev);

	/* check endpoint configuration space is accessible */
	while (time_before(jiffies, ep_up_timeout)) {
		if (readl_relaxed(dev->conf) != PCIE_LINK_DOWN)
			break;
		usleep_range(EP_UP_TIMEOUT_US_MIN, EP_UP_TIMEOUT_US_MAX);
	}

	if (readl_relaxed(dev->conf) != PCIE_LINK_DOWN) {
		PCIE_DBG(dev,
			"PCIe: RC%d: endpoint config space is accessible\n",
			dev->rc_idx);
	} else {
		PCIE_ERR(dev,
			"PCIe: RC%d: endpoint config space is not accessible\n",
			dev->rc_idx);
		dev->link_status = MSM_PCIE_LINK_DISABLED;
		dev->power_on = false;
		dev->link_turned_off_counter++;
		ret = -ENODEV;
		goto link_fail;
	}

	if (!dev->msi_gicm_addr)
		msm_pcie_config_msi_controller(dev);

	if (dev->enumerated) {
		pci_walk_bus(dev->dev->bus, &msm_pcie_config_device, dev);
		msm_pcie_config_link_pm_rc(dev, dev->dev, true);
	}

	goto out;

link_fail:
	if (msm_pcie_keep_resources_on & BIT(dev->rc_idx))
		goto out;

	if (dev->gpio[MSM_PCIE_GPIO_EP].num)
		gpio_set_value(dev->gpio[MSM_PCIE_GPIO_EP].num,
				1 - dev->gpio[MSM_PCIE_GPIO_EP].on);

	if (dev->max_link_speed != GEN3_SPEED) {
		msm_pcie_write_reg(dev->phy,
			PCIE_N_SW_RESET(dev->rc_idx), 0x1);
		msm_pcie_write_reg(dev->phy,
			PCIE_N_POWER_DOWN_CONTROL(dev->rc_idx), 0);
	}

	msm_pcie_pipe_clk_deinit(dev);
	msm_pcie_clk_deinit(dev);
clk_fail:
	msm_pcie_vreg_deinit(dev);
out:
	mutex_unlock(&dev->setup_lock);

	PCIE_DBG(dev, "RC%d: exit\n", dev->rc_idx);

	return ret;
}

static void msm_pcie_disable(struct msm_pcie_dev_t *dev, u32 options)
{
	PCIE_DBG(dev, "RC%d: entry\n", dev->rc_idx);

	mutex_lock(&dev->setup_lock);

	if (!dev->power_on) {
		PCIE_DBG(dev,
			"PCIe: the link of RC%d is already power down.\n",
			dev->rc_idx);
		mutex_unlock(&dev->setup_lock);
		return;
	}

	dev->link_status = MSM_PCIE_LINK_DISABLED;
	dev->power_on = false;
	dev->link_turned_off_counter++;

	PCIE_INFO(dev, "PCIe: Assert the reset of endpoint of RC%d.\n",
		dev->rc_idx);

	gpio_set_value(dev->gpio[MSM_PCIE_GPIO_PERST].num,
				dev->gpio[MSM_PCIE_GPIO_PERST].on);

	if (dev->max_link_speed != GEN3_SPEED) {
		msm_pcie_write_reg(dev->phy,
			PCIE_N_SW_RESET(dev->rc_idx), 0x1);
		msm_pcie_write_reg(dev->phy,
			PCIE_N_POWER_DOWN_CONTROL(dev->rc_idx), 0);
	}

	if (options & PM_CLK) {
		msm_pcie_write_mask(dev->parf + PCIE20_PARF_PHY_CTRL, 0,
					BIT(0));
		msm_pcie_clk_deinit(dev);
	}

	if (options & PM_VREG)
		msm_pcie_vreg_deinit(dev);

	if (options & PM_PIPE_CLK)
		msm_pcie_pipe_clk_deinit(dev);

	if (dev->gpio[MSM_PCIE_GPIO_EP].num)
		gpio_set_value(dev->gpio[MSM_PCIE_GPIO_EP].num,
				1 - dev->gpio[MSM_PCIE_GPIO_EP].on);

	mutex_unlock(&dev->setup_lock);

	PCIE_DBG(dev, "RC%d: exit\n", dev->rc_idx);
}

static void msm_pcie_config_ep_aer(struct msm_pcie_dev_t *dev,
				struct msm_pcie_device_info *ep_dev_info)
{
	u32 val;
	void __iomem *ep_base = ep_dev_info->conf_base;
	u32 current_offset = readl_relaxed(ep_base + PCIE_CAP_PTR_OFFSET) &
						0xff;

	while (current_offset) {
		if (msm_pcie_check_align(dev, current_offset))
			return;

		val = readl_relaxed(ep_base + current_offset);
		if ((val & 0xff) == PCIE20_CAP_ID) {
			ep_dev_info->dev_ctrlstts_offset =
				current_offset + 0x8;
			break;
		}
		current_offset = (val >> 8) & 0xff;
	}

	if (!ep_dev_info->dev_ctrlstts_offset) {
		PCIE_DBG(dev,
			"RC%d endpoint does not support PCIe cap registers\n",
			dev->rc_idx);
		return;
	}

	PCIE_DBG2(dev, "RC%d: EP dev_ctrlstts_offset: 0x%x\n",
		dev->rc_idx, ep_dev_info->dev_ctrlstts_offset);

	/* Enable AER on EP */
	msm_pcie_write_mask(ep_base + ep_dev_info->dev_ctrlstts_offset, 0,
				BIT(3)|BIT(2)|BIT(1)|BIT(0));

	PCIE_DBG(dev, "EP's PCIE20_CAP_DEVCTRLSTATUS:0x%x\n",
		readl_relaxed(ep_base + ep_dev_info->dev_ctrlstts_offset));
}

static int msm_pcie_config_device_table(struct device *dev, void *pdev)
{
	struct pci_dev *pcidev = to_pci_dev(dev);
	struct msm_pcie_dev_t *pcie_dev = (struct msm_pcie_dev_t *) pdev;
	struct msm_pcie_device_info *dev_table_t = pcie_dev->pcidev_table;
	struct resource *axi_conf = pcie_dev->res[MSM_PCIE_RES_CONF].resource;
	int ret = 0;
	u32 rc_idx = pcie_dev->rc_idx;
	u32 i, index;
	u32 bdf = 0;
	u8 type;
	u32 h_type;
	u32 bme;

	if (!pcidev) {
		PCIE_ERR(pcie_dev,
			"PCIe: Did not find PCI device in list for RC%d.\n",
			pcie_dev->rc_idx);
		return -ENODEV;
	}

	PCIE_DBG(pcie_dev,
		"PCI device found: vendor-id:0x%x device-id:0x%x\n",
		pcidev->vendor, pcidev->device);

	if (!pcidev->bus->number)
		return ret;

	bdf = BDF_OFFSET(pcidev->bus->number, pcidev->devfn);
	type = pcidev->bus->number == 1 ?
		PCIE20_CTRL1_TYPE_CFG0 : PCIE20_CTRL1_TYPE_CFG1;

	for (i = 0; i < (MAX_RC_NUM * MAX_DEVICE_NUM); i++) {
		if (msm_pcie_dev_tbl[i].bdf == bdf &&
			!msm_pcie_dev_tbl[i].dev) {
			for (index = 0; index < MAX_DEVICE_NUM; index++) {
				if (dev_table_t[index].bdf == bdf) {
					msm_pcie_dev_tbl[i].dev = pcidev;
					msm_pcie_dev_tbl[i].domain = rc_idx;
					msm_pcie_dev_tbl[i].conf_base =
						pcie_dev->conf + index * SZ_4K;
					msm_pcie_dev_tbl[i].phy_address =
						axi_conf->start + index * SZ_4K;

					dev_table_t[index].dev = pcidev;
					dev_table_t[index].domain = rc_idx;
					dev_table_t[index].conf_base =
						pcie_dev->conf + index * SZ_4K;
					dev_table_t[index].phy_address =
						axi_conf->start + index * SZ_4K;

					msm_pcie_iatu_config(pcie_dev, index,
						type,
						dev_table_t[index].phy_address,
						dev_table_t[index].phy_address
						+ SZ_4K - 1,
						bdf);

					h_type = readl_relaxed(
						dev_table_t[index].conf_base +
						PCIE20_HEADER_TYPE);

					bme = readl_relaxed(
						dev_table_t[index].conf_base +
						PCIE20_COMMAND_STATUS);

					if (h_type & (1 << 16)) {
						pci_write_config_dword(pcidev,
							PCIE20_COMMAND_STATUS,
							bme | 0x06);
					} else {
						pcie_dev->num_ep++;
						dev_table_t[index].registered =
							false;
					}

					if (pcie_dev->num_ep > 1)
						pcie_dev->pending_ep_reg = true;

					if (pcie_dev->aer_enable)
						msm_pcie_config_ep_aer(pcie_dev,
							&dev_table_t[index]);

					break;
				}
			}
			if (index == MAX_DEVICE_NUM) {
				PCIE_ERR(pcie_dev,
					"RC%d PCI device table is full.\n",
					rc_idx);
				ret = index;
			} else {
				break;
			}
		} else if (msm_pcie_dev_tbl[i].bdf == bdf &&
			pcidev == msm_pcie_dev_tbl[i].dev) {
			break;
		}
	}
	if (i == MAX_RC_NUM * MAX_DEVICE_NUM) {
		PCIE_ERR(pcie_dev,
			"Global PCI device table is full: %d elements.\n",
			i);
		PCIE_ERR(pcie_dev,
			"Bus number is 0x%x\nDevice number is 0x%x\n",
			pcidev->bus->number, pcidev->devfn);
		ret = i;
	}
	return ret;
}

static void msm_pcie_configure_sid(struct msm_pcie_dev_t *pcie_dev,
				struct pci_dev *dev)
{
	u32 offset;
	u32 sid;
	u32 bdf;
	int ret;

	ret = iommu_fwspec_get_id(&dev->dev, &sid);
	if (ret) {
		PCIE_DBG(pcie_dev,
			"PCIe: RC%d: Device does not have a SID\n",
			pcie_dev->rc_idx);
		return;
	}

	PCIE_DBG(pcie_dev,
		"PCIe: RC%d: Device SID: 0x%x\n",
		pcie_dev->rc_idx, sid);

	bdf = BDF_OFFSET(dev->bus->number, dev->devfn);
	offset = (sid - pcie_dev->smmu_sid_base) * 4;

	if (offset >= MAX_SHORT_BDF_NUM * 4) {
		PCIE_ERR(pcie_dev,
			"PCIe: RC%d: Invalid SID offset: 0x%x. Should be less than 0x%x\n",
			pcie_dev->rc_idx, offset, MAX_SHORT_BDF_NUM * 4);
		return;
	}

	msm_pcie_write_reg(pcie_dev->parf, PCIE20_PARF_BDF_TRANSLATE_CFG, 0);
	msm_pcie_write_reg(pcie_dev->parf, PCIE20_PARF_SID_OFFSET, 0);
	msm_pcie_write_reg(pcie_dev->parf,
		PCIE20_PARF_BDF_TRANSLATE_N + offset, bdf >> 16);
}

int msm_pcie_enumerate(u32 rc_idx)
{
	int ret = 0, bus_ret = 0, scan_ret = 0;
	struct msm_pcie_dev_t *dev = &msm_pcie_dev[rc_idx];

	mutex_lock(&dev->enumerate_lock);

	PCIE_DBG(dev, "Enumerate RC%d\n", rc_idx);

	if (!dev->drv_ready) {
		PCIE_DBG(dev, "RC%d has not been successfully probed yet\n",
			rc_idx);
		ret = -EPROBE_DEFER;
		goto out;
	}

	if (!dev->enumerated) {
		ret = msm_pcie_enable(dev, PM_ALL);

		/* kick start ARM PCI configuration framework */
		if (!ret) {
			struct pci_dev *pcidev = NULL;
			bool found = false;
			struct pci_bus *bus;
			resource_size_t iobase = 0;
			u32 ids = readl_relaxed(msm_pcie_dev[rc_idx].dm_core);
			u32 vendor_id = ids & 0xffff;
			u32 device_id = (ids & 0xffff0000) >> 16;
			LIST_HEAD(res);

			PCIE_DBG(dev, "vendor-id:0x%x device_id:0x%x\n",
					vendor_id, device_id);

			ret = of_pci_get_host_bridge_resources(
						dev->pdev->dev.of_node,
						0, 0xff, &res, &iobase);
			if (ret) {
				PCIE_ERR(dev,
					"PCIe: failed to get host bridge resources for RC%d: %d\n",
					dev->rc_idx, ret);
				goto out;
			}

			bus = pci_create_root_bus(&dev->pdev->dev, 0,
						&msm_pcie_ops, dev, &res);
			if (!bus) {
				PCIE_ERR(dev,
					"PCIe: failed to create root bus for RC%d\n",
					dev->rc_idx);
				ret = -ENOMEM;
				goto out;
			}

			scan_ret = pci_scan_child_bus(bus);
			PCIE_DBG(dev,
				"PCIe: RC%d: The max subordinate bus number discovered is %d\n",
				dev->rc_idx, scan_ret);

			msm_pcie_fixup_irqs(dev);
			pci_assign_unassigned_bus_resources(bus);
			pci_bus_add_devices(bus);

			dev->enumerated = true;

			msm_pcie_write_mask(dev->dm_core +
				PCIE20_COMMAND_STATUS, 0, BIT(2)|BIT(1));

			if (dev->cpl_timeout && dev->bridge_found)
				msm_pcie_write_reg_field(dev->dm_core,
					PCIE20_DEVICE_CONTROL2_STATUS2,
					0xf, dev->cpl_timeout);

			if (dev->shadow_en) {
				u32 val = readl_relaxed(dev->dm_core +
						PCIE20_COMMAND_STATUS);
				PCIE_DBG(dev, "PCIE20_COMMAND_STATUS:0x%x\n",
					val);
				dev->rc_shadow[PCIE20_COMMAND_STATUS / 4] = val;
			}

			do {
				pcidev = pci_get_device(vendor_id,
					device_id, pcidev);
				if (pcidev && (&msm_pcie_dev[rc_idx] ==
					(struct msm_pcie_dev_t *)
					PCIE_BUS_PRIV_DATA(pcidev->bus))) {
					msm_pcie_dev[rc_idx].dev = pcidev;
					found = true;
					PCIE_DBG(&msm_pcie_dev[rc_idx],
						"PCI device is found for RC%d\n",
						rc_idx);
				}
			} while (!found && pcidev);

			if (!pcidev) {
				PCIE_ERR(dev,
					"PCIe: Did not find PCI device for RC%d.\n",
					dev->rc_idx);
				ret = -ENODEV;
				goto out;
			}

			bus_ret = bus_for_each_dev(&pci_bus_type, NULL, dev,
					&msm_pcie_config_device_table);

			if (bus_ret) {
				PCIE_ERR(dev,
					"PCIe: Failed to set up device table for RC%d\n",
					dev->rc_idx);
				ret = -ENODEV;
				goto out;
			}

			msm_pcie_config_link_pm_rc(dev, dev->dev, true);
		} else {
			PCIE_ERR(dev, "PCIe: failed to enable RC%d.\n",
				dev->rc_idx);
		}
	} else {
		PCIE_ERR(dev, "PCIe: RC%d has already been enumerated.\n",
			dev->rc_idx);
	}

out:
	mutex_unlock(&dev->enumerate_lock);

	return ret;
}
EXPORT_SYMBOL(msm_pcie_enumerate);

static void msm_pcie_notify_client(struct msm_pcie_dev_t *dev,
					enum msm_pcie_event event)
{
	if (dev->event_reg && dev->event_reg->callback &&
		(dev->event_reg->events & event)) {
		struct msm_pcie_notify *notify = &dev->event_reg->notify;

		notify->event = event;
		notify->user = dev->event_reg->user;
		PCIE_DBG(dev, "PCIe: callback RC%d for event %d\n",
			dev->rc_idx, event);
		dev->event_reg->callback(notify);

		if ((dev->event_reg->options & MSM_PCIE_CONFIG_NO_RECOVERY) &&
				(event == MSM_PCIE_EVENT_LINKDOWN)) {
			dev->user_suspend = true;
			PCIE_DBG(dev,
				"PCIe: Client of RC%d will recover the link later.\n",
				dev->rc_idx);
			return;
		}
	} else {
		PCIE_DBG2(dev,
			"PCIe: Client of RC%d does not have registration for event %d\n",
			dev->rc_idx, event);
	}
}

static void handle_wake_func(struct work_struct *work)
{
	int i, ret;
	struct msm_pcie_dev_t *dev = container_of(work, struct msm_pcie_dev_t,
					handle_wake_work);

	PCIE_DBG(dev, "PCIe: Wake work for RC%d\n", dev->rc_idx);

	mutex_lock(&dev->recovery_lock);

	if (!dev->enumerated) {
		PCIE_DBG(dev,
			"PCIe: Start enumeration for RC%d upon the wake from endpoint.\n",
			dev->rc_idx);

		ret = msm_pcie_enumerate(dev->rc_idx);
		if (ret) {
			PCIE_ERR(dev,
				"PCIe: failed to enable RC%d upon wake request from the device.\n",
				dev->rc_idx);
			goto out;
		}

		if (dev->num_ep > 1) {
			for (i = 0; i < MAX_DEVICE_NUM; i++) {
				dev->event_reg = dev->pcidev_table[i].event_reg;

				if ((dev->link_status == MSM_PCIE_LINK_ENABLED)
					&& dev->event_reg &&
					dev->event_reg->callback &&
					(dev->event_reg->events &
					MSM_PCIE_EVENT_LINKUP)) {
					struct msm_pcie_notify *notify =
						&dev->event_reg->notify;
					notify->event = MSM_PCIE_EVENT_LINKUP;
					notify->user = dev->event_reg->user;
					PCIE_DBG(dev,
						"PCIe: Linkup callback for RC%d after enumeration is successful in wake IRQ handling\n",
						dev->rc_idx);
					dev->event_reg->callback(notify);
				}
			}
		} else {
			if ((dev->link_status == MSM_PCIE_LINK_ENABLED) &&
				dev->event_reg && dev->event_reg->callback &&
				(dev->event_reg->events &
				MSM_PCIE_EVENT_LINKUP)) {
				struct msm_pcie_notify *notify =
						&dev->event_reg->notify;
				notify->event = MSM_PCIE_EVENT_LINKUP;
				notify->user = dev->event_reg->user;
				PCIE_DBG(dev,
					"PCIe: Linkup callback for RC%d after enumeration is successful in wake IRQ handling\n",
					dev->rc_idx);
				dev->event_reg->callback(notify);
			} else {
				PCIE_DBG(dev,
					"PCIe: Client of RC%d does not have registration for linkup event.\n",
					dev->rc_idx);
			}
		}
		goto out;
	} else {
		PCIE_ERR(dev,
			"PCIe: The enumeration for RC%d has already been done.\n",
			dev->rc_idx);
		goto out;
	}

out:
	mutex_unlock(&dev->recovery_lock);
}

static irqreturn_t handle_aer_irq(int irq, void *data)
{
	struct msm_pcie_dev_t *dev = data;

	int corr_val = 0, uncorr_val = 0, rc_err_status = 0;
	int ep_corr_val = 0, ep_uncorr_val = 0;
	int rc_dev_ctrlstts = 0, ep_dev_ctrlstts = 0;
	u32 ep_dev_ctrlstts_offset = 0;
	int i, j, ep_src_bdf = 0;
	void __iomem *ep_base = NULL;

	PCIE_DBG2(dev,
		"AER Interrupt handler fired for RC%d irq %d\nrc_corr_counter: %lu\nrc_non_fatal_counter: %lu\nrc_fatal_counter: %lu\nep_corr_counter: %lu\nep_non_fatal_counter: %lu\nep_fatal_counter: %lu\n",
		dev->rc_idx, irq, dev->rc_corr_counter,
		dev->rc_non_fatal_counter, dev->rc_fatal_counter,
		dev->ep_corr_counter, dev->ep_non_fatal_counter,
		dev->ep_fatal_counter);

	uncorr_val = readl_relaxed(dev->dm_core +
				PCIE20_AER_UNCORR_ERR_STATUS_REG);
	corr_val = readl_relaxed(dev->dm_core +
				PCIE20_AER_CORR_ERR_STATUS_REG);
	rc_err_status = readl_relaxed(dev->dm_core +
				PCIE20_AER_ROOT_ERR_STATUS_REG);
	rc_dev_ctrlstts = readl_relaxed(dev->dm_core +
				PCIE20_CAP_DEVCTRLSTATUS);

	if (uncorr_val)
		PCIE_DBG(dev, "RC's PCIE20_AER_UNCORR_ERR_STATUS_REG:0x%x\n",
				uncorr_val);
	if (corr_val && (dev->rc_corr_counter < corr_counter_limit))
		PCIE_DBG(dev, "RC's PCIE20_AER_CORR_ERR_STATUS_REG:0x%x\n",
				corr_val);

	if ((rc_dev_ctrlstts >> 18) & 0x1)
		dev->rc_fatal_counter++;
	if ((rc_dev_ctrlstts >> 17) & 0x1)
		dev->rc_non_fatal_counter++;
	if ((rc_dev_ctrlstts >> 16) & 0x1)
		dev->rc_corr_counter++;

	msm_pcie_write_mask(dev->dm_core + PCIE20_CAP_DEVCTRLSTATUS, 0,
				BIT(18)|BIT(17)|BIT(16));

	if (dev->link_status == MSM_PCIE_LINK_DISABLED) {
		PCIE_DBG2(dev, "RC%d link is down\n", dev->rc_idx);
		goto out;
	}

	for (i = 0; i < 2; i++) {
		if (i)
			ep_src_bdf = readl_relaxed(dev->dm_core +
				PCIE20_AER_ERR_SRC_ID_REG) & ~0xffff;
		else
			ep_src_bdf = (readl_relaxed(dev->dm_core +
				PCIE20_AER_ERR_SRC_ID_REG) & 0xffff) << 16;

		if (!ep_src_bdf)
			continue;

		for (j = 0; j < MAX_DEVICE_NUM; j++) {
			if (ep_src_bdf == dev->pcidev_table[j].bdf) {
				PCIE_DBG2(dev,
					"PCIe: %s Error from Endpoint: %02x:%02x.%01x\n",
					i ? "Uncorrectable" : "Correctable",
					dev->pcidev_table[j].bdf >> 24,
					dev->pcidev_table[j].bdf >> 19 & 0x1f,
					dev->pcidev_table[j].bdf >> 16 & 0x07);
				ep_base = dev->pcidev_table[j].conf_base;
				ep_dev_ctrlstts_offset = dev->
					pcidev_table[j].dev_ctrlstts_offset;
				break;
			}
		}

		if (!ep_base) {
			PCIE_ERR(dev,
				"PCIe: RC%d no endpoint found for reported error\n",
				dev->rc_idx);
			goto out;
		}

		ep_uncorr_val = readl_relaxed(ep_base +
					PCIE20_AER_UNCORR_ERR_STATUS_REG);
		ep_corr_val = readl_relaxed(ep_base +
					PCIE20_AER_CORR_ERR_STATUS_REG);
		ep_dev_ctrlstts = readl_relaxed(ep_base +
					ep_dev_ctrlstts_offset);

		if (ep_uncorr_val)
			PCIE_DBG(dev,
				"EP's PCIE20_AER_UNCORR_ERR_STATUS_REG:0x%x\n",
				ep_uncorr_val);
		if (ep_corr_val && (dev->ep_corr_counter < corr_counter_limit))
			PCIE_DBG(dev,
				"EP's PCIE20_AER_CORR_ERR_STATUS_REG:0x%x\n",
				ep_corr_val);

		if ((ep_dev_ctrlstts >> 18) & 0x1)
			dev->ep_fatal_counter++;
		if ((ep_dev_ctrlstts >> 17) & 0x1)
			dev->ep_non_fatal_counter++;
		if ((ep_dev_ctrlstts >> 16) & 0x1)
			dev->ep_corr_counter++;

		msm_pcie_write_mask(ep_base + ep_dev_ctrlstts_offset, 0,
					BIT(18)|BIT(17)|BIT(16));

		msm_pcie_write_reg_field(ep_base,
				PCIE20_AER_UNCORR_ERR_STATUS_REG,
				0x3fff031, 0x3fff031);
		msm_pcie_write_reg_field(ep_base,
				PCIE20_AER_CORR_ERR_STATUS_REG,
				0xf1c1, 0xf1c1);
	}
out:
	if (((dev->rc_corr_counter < corr_counter_limit) &&
		(dev->ep_corr_counter < corr_counter_limit)) ||
		uncorr_val || ep_uncorr_val)
		PCIE_DBG(dev, "RC's PCIE20_AER_ROOT_ERR_STATUS_REG:0x%x\n",
				rc_err_status);
	msm_pcie_write_reg_field(dev->dm_core,
			PCIE20_AER_UNCORR_ERR_STATUS_REG,
			0x3fff031, 0x3fff031);
	msm_pcie_write_reg_field(dev->dm_core,
			PCIE20_AER_CORR_ERR_STATUS_REG,
			0xf1c1, 0xf1c1);
	msm_pcie_write_reg_field(dev->dm_core,
			PCIE20_AER_ROOT_ERR_STATUS_REG,
			0x7f, 0x7f);

	return IRQ_HANDLED;
}

static irqreturn_t handle_wake_irq(int irq, void *data)
{
	struct msm_pcie_dev_t *dev = data;
	unsigned long irqsave_flags;
	int i;

	spin_lock_irqsave(&dev->wakeup_lock, irqsave_flags);

	dev->wake_counter++;
	PCIE_DBG(dev, "PCIe: No. %ld wake IRQ for RC%d\n",
			dev->wake_counter, dev->rc_idx);

	PCIE_DBG2(dev, "PCIe WAKE is asserted by Endpoint of RC%d\n",
		dev->rc_idx);

	if (!dev->enumerated && !(dev->boot_option &
		MSM_PCIE_NO_WAKE_ENUMERATION)) {
		PCIE_DBG(dev, "Start enumerating RC%d\n", dev->rc_idx);
		schedule_work(&dev->handle_wake_work);
	} else {
		PCIE_DBG2(dev, "Wake up RC%d\n", dev->rc_idx);
		__pm_stay_awake(&dev->ws);
		__pm_relax(&dev->ws);

		if (dev->num_ep > 1) {
			for (i = 0; i < MAX_DEVICE_NUM; i++) {
				dev->event_reg =
					dev->pcidev_table[i].event_reg;
				msm_pcie_notify_client(dev,
					MSM_PCIE_EVENT_WAKEUP);
			}
		} else {
			msm_pcie_notify_client(dev, MSM_PCIE_EVENT_WAKEUP);
		}
	}

	spin_unlock_irqrestore(&dev->wakeup_lock, irqsave_flags);

	return IRQ_HANDLED;
}

static irqreturn_t handle_linkdown_irq(int irq, void *data)
{
	struct msm_pcie_dev_t *dev = data;
	int i;

	dev->linkdown_counter++;

	PCIE_DBG(dev,
		"PCIe: No. %ld linkdown IRQ for RC%d.\n",
		dev->linkdown_counter, dev->rc_idx);

	if (!dev->enumerated || dev->link_status != MSM_PCIE_LINK_ENABLED) {
		PCIE_DBG(dev,
			"PCIe:Linkdown IRQ for RC%d when the link is not enabled\n",
			dev->rc_idx);
	} else if (dev->suspending) {
		PCIE_DBG(dev,
			"PCIe:the link of RC%d is suspending.\n",
			dev->rc_idx);
	} else {
		dev->link_status = MSM_PCIE_LINK_DISABLED;
		dev->shadow_en = false;

		if (dev->linkdown_panic)
			panic("User has chosen to panic on linkdown\n");

		/* assert PERST */
		if (!(msm_pcie_keep_resources_on & BIT(dev->rc_idx)))
			gpio_set_value(dev->gpio[MSM_PCIE_GPIO_PERST].num,
					dev->gpio[MSM_PCIE_GPIO_PERST].on);

		PCIE_ERR(dev, "PCIe link is down for RC%d\n", dev->rc_idx);

		if (dev->num_ep > 1) {
			for (i = 0; i < MAX_DEVICE_NUM; i++) {
				dev->event_reg =
					dev->pcidev_table[i].event_reg;
				msm_pcie_notify_client(dev,
					MSM_PCIE_EVENT_LINKDOWN);
			}
		} else {
			msm_pcie_notify_client(dev, MSM_PCIE_EVENT_LINKDOWN);
		}
	}

	return IRQ_HANDLED;
}

static irqreturn_t handle_msi_irq(int irq, void *data)
{
	int i, j;
	unsigned long val;
	struct msm_pcie_dev_t *dev = data;
	void __iomem *ctrl_status;

	PCIE_DUMP(dev, "irq: %d\n", irq);

	/*
	 * check for set bits, clear it by setting that bit
	 * and trigger corresponding irq
	 */
	for (i = 0; i < PCIE20_MSI_CTRL_MAX; i++) {
		ctrl_status = dev->dm_core +
				PCIE20_MSI_CTRL_INTR_STATUS + (i * 12);

		val = readl_relaxed(ctrl_status);
		while (val) {
			j = find_first_bit(&val, 32);
			writel_relaxed(BIT(j), ctrl_status);
			/* ensure that interrupt is cleared (acked) */
			wmb();
			generic_handle_irq(
			   irq_find_mapping(dev->irq_domain, (j + (32*i)))
			   );
			val = readl_relaxed(ctrl_status);
		}
	}

	return IRQ_HANDLED;
}

static irqreturn_t handle_global_irq(int irq, void *data)
{
	int i;
	struct msm_pcie_dev_t *dev = data;
	unsigned long irqsave_flags;
	u32 status = 0;

	spin_lock_irqsave(&dev->irq_lock, irqsave_flags);

	if (dev->suspending) {
		PCIE_DBG2(dev,
			"PCIe: RC%d is currently suspending.\n",
			dev->rc_idx);
		spin_unlock_irqrestore(&dev->irq_lock, irqsave_flags);
		return IRQ_HANDLED;
	}

	status = readl_relaxed(dev->parf + PCIE20_PARF_INT_ALL_STATUS) &
			readl_relaxed(dev->parf + PCIE20_PARF_INT_ALL_MASK);

	msm_pcie_write_mask(dev->parf + PCIE20_PARF_INT_ALL_CLEAR, 0, status);

	PCIE_DBG2(dev, "RC%d: Global IRQ %d received: 0x%x\n",
		dev->rc_idx, irq, status);

	for (i = 0; i <= MSM_PCIE_INT_EVT_MAX; i++) {
		if (status & BIT(i)) {
			switch (i) {
			case MSM_PCIE_INT_EVT_LINK_DOWN:
				PCIE_DBG(dev,
					"PCIe: RC%d: handle linkdown event.\n",
					dev->rc_idx);
				handle_linkdown_irq(irq, data);
				break;
			case MSM_PCIE_INT_EVT_AER_LEGACY:
				PCIE_DBG(dev,
					"PCIe: RC%d: AER legacy event.\n",
					dev->rc_idx);
				handle_aer_irq(irq, data);
				break;
			case MSM_PCIE_INT_EVT_AER_ERR:
				PCIE_DBG(dev,
					"PCIe: RC%d: AER event.\n",
					dev->rc_idx);
				handle_aer_irq(irq, data);
				break;
			default:
				PCIE_DUMP(dev,
					"PCIe: RC%d: Unexpected event %d is caught!\n",
					dev->rc_idx, i);
			}
		}
	}

	spin_unlock_irqrestore(&dev->irq_lock, irqsave_flags);

	return IRQ_HANDLED;
}

static void msm_pcie_unmap_qgic_addr(struct msm_pcie_dev_t *dev,
					struct pci_dev *pdev,
					struct msi_desc *entry)
{
	struct iommu_domain *domain = iommu_get_domain_for_dev(&pdev->dev);
	int bypass_en = 0;

	if (!domain) {
		PCIE_DBG(dev,
			"PCIe: RC%d: client does not have an iommu domain\n",
			dev->rc_idx);
		return;
	}

	iommu_domain_get_attr(domain, DOMAIN_ATTR_S1_BYPASS, &bypass_en);
	if (!bypass_en) {
		dma_addr_t iova = entry->msg.address_lo;

		PCIE_DBG(dev, "PCIe: RC%d: unmap QGIC MSI IOVA\n", dev->rc_idx);

		dma_unmap_resource(&pdev->dev, iova, PAGE_SIZE,
				DMA_BIDIRECTIONAL, 0);
	}
}

static void msm_pcie_destroy_irq(struct msi_desc *entry, unsigned int irq)
{
	int pos;
	struct msm_pcie_dev_t *dev;
	struct pci_dev *pdev = msi_desc_to_pci_dev(entry);

	if (!pdev) {
		pr_err("PCIe: pci device is null. IRQ:%d\n", irq);
		return;
	}

	dev = PCIE_BUS_PRIV_DATA(pdev->bus);
	if (!dev) {
		pr_err("PCIe: could not find RC. IRQ:%d\n", irq);
		return;
	}

	if (dev->msi_gicm_addr) {
		int firstirq = entry->irq;
		u32 nvec = (1 << entry->msi_attrib.multiple);

		PCIE_DBG(dev, "destroy QGIC based irq %d\n", irq);

		if (irq < firstirq || irq > firstirq + nvec - 1) {
			PCIE_ERR(dev,
				"Could not find irq: %d in RC%d MSI table\n",
				irq, dev->rc_idx);
			return;
		}
		if (irq == firstirq + nvec - 1)
			msm_pcie_unmap_qgic_addr(dev, pdev, entry);
		pos = irq - firstirq;
	} else {
		PCIE_DBG(dev, "destroy default MSI irq %d\n", irq);
		pos = irq - irq_find_mapping(dev->irq_domain, 0);
	}

	PCIE_DBG(dev, "RC%d\n", dev->rc_idx);

	PCIE_DBG(dev, "Before clear_bit pos:%d msi_irq_in_use:%ld\n",
		pos, *dev->msi_irq_in_use);
	clear_bit(pos, dev->msi_irq_in_use);
	PCIE_DBG(dev, "After clear_bit pos:%d msi_irq_in_use:%ld\n",
		pos, *dev->msi_irq_in_use);
}

/* hookup to linux pci msi framework */
void arch_teardown_msi_irq(unsigned int irq)
{
	struct msi_desc *entry = irq_get_msi_desc(irq);

	PCIE_GEN_DBG("irq %d deallocated\n", irq);

	if (entry)
		msm_pcie_destroy_irq(entry, irq);
}

void arch_teardown_msi_irqs(struct pci_dev *dev)
{
	struct msi_desc *entry;
	struct msm_pcie_dev_t *pcie_dev = PCIE_BUS_PRIV_DATA(dev->bus);

	PCIE_DBG(pcie_dev, "RC:%d EP: vendor_id:0x%x device_id:0x%x\n",
		pcie_dev->rc_idx, dev->vendor, dev->device);

	pcie_dev->use_msi = false;

	list_for_each_entry(entry, &dev->dev.msi_list, list) {
		int i, nvec;

		if (entry->irq == 0)
			continue;
		nvec = 1 << entry->msi_attrib.multiple;
		for (i = 0; i < nvec; i++)
			msm_pcie_destroy_irq(entry, entry->irq + i);
	}
}

static void msm_pcie_msi_nop(struct irq_data *d)
{
}

static struct irq_chip pcie_msi_chip = {
	.name = "msm-pcie-msi",
	.irq_ack = msm_pcie_msi_nop,
	.irq_enable = unmask_msi_irq,
	.irq_disable = mask_msi_irq,
	.irq_mask = mask_msi_irq,
	.irq_unmask = unmask_msi_irq,
};

static int msm_pcie_create_irq(struct msm_pcie_dev_t *dev)
{
	int irq, pos;

	PCIE_DBG(dev, "RC%d\n", dev->rc_idx);

again:
	pos = find_first_zero_bit(dev->msi_irq_in_use, PCIE_MSI_NR_IRQS);

	if (pos >= PCIE_MSI_NR_IRQS)
		return -ENOSPC;

	PCIE_DBG(dev, "pos:%d msi_irq_in_use:%ld\n", pos, *dev->msi_irq_in_use);

	if (test_and_set_bit(pos, dev->msi_irq_in_use))
		goto again;
	else
		PCIE_DBG(dev, "test_and_set_bit is successful pos=%d\n", pos);

	irq = irq_create_mapping(dev->irq_domain, pos);
	if (!irq)
		return -EINVAL;

	return irq;
}

static int arch_setup_msi_irq_default(struct pci_dev *pdev,
		struct msi_desc *desc, int nvec)
{
	int irq;
	struct msi_msg msg;
	struct msm_pcie_dev_t *dev = PCIE_BUS_PRIV_DATA(pdev->bus);

	PCIE_DBG(dev, "RC%d\n", dev->rc_idx);

	irq = msm_pcie_create_irq(dev);

	PCIE_DBG(dev, "IRQ %d is allocated.\n", irq);

	if (irq < 0)
		return irq;

	PCIE_DBG(dev, "irq %d allocated\n", irq);

	irq_set_chip_data(irq, pdev);
	irq_set_msi_desc(irq, desc);

	/* write msi vector and data */
	msg.address_hi = 0;
	msg.address_lo = MSM_PCIE_MSI_PHY;
	msg.data = irq - irq_find_mapping(dev->irq_domain, 0);
	write_msi_msg(irq, &msg);

	return 0;
}

static int msm_pcie_create_irq_qgic(struct msm_pcie_dev_t *dev)
{
	int irq, pos;

	PCIE_DBG(dev, "RC%d\n", dev->rc_idx);

again:
	pos = find_first_zero_bit(dev->msi_irq_in_use, PCIE_MSI_NR_IRQS);

	if (pos >= PCIE_MSI_NR_IRQS)
		return -ENOSPC;

	PCIE_DBG(dev, "pos:%d msi_irq_in_use:%ld\n", pos, *dev->msi_irq_in_use);

	if (test_and_set_bit(pos, dev->msi_irq_in_use))
		goto again;
	else
		PCIE_DBG(dev, "test_and_set_bit is successful pos=%d\n", pos);

	if (pos >= MSM_PCIE_MAX_MSI) {
		PCIE_ERR(dev,
			"PCIe: RC%d: pos %d is not less than %d\n",
			dev->rc_idx, pos, MSM_PCIE_MAX_MSI);
		return MSM_PCIE_ERROR;
	}

	irq = dev->msi[pos].num;
	if (!irq) {
		PCIE_ERR(dev, "PCIe: RC%d failed to create QGIC MSI IRQ.\n",
			dev->rc_idx);
		return -EINVAL;
	}

	return irq;
}

static int msm_pcie_map_qgic_addr(struct msm_pcie_dev_t *dev,
					struct pci_dev *pdev,
					struct msi_msg *msg)
{
	struct iommu_domain *domain = iommu_get_domain_for_dev(&pdev->dev);
	int bypass_en = 0;
	dma_addr_t iova;

	msg->address_hi = 0;
	msg->address_lo = dev->msi_gicm_addr;

	if (!domain) {
		PCIE_DBG(dev,
			"PCIe: RC%d: client does not have an iommu domain\n",
			dev->rc_idx);
		return 0;
	}

	iommu_domain_get_attr(domain, DOMAIN_ATTR_S1_BYPASS, &bypass_en);

	PCIE_DBG(dev,
		"PCIe: RC%d: Stage 1 is %s for endpoint: %04x:%02x\n",
		dev->rc_idx, bypass_en ? "bypass" : "enabled",
		pdev->bus->number, pdev->devfn);

	if (bypass_en)
		return 0;

	iova = dma_map_resource(&pdev->dev, dev->msi_gicm_addr, PAGE_SIZE,
				DMA_BIDIRECTIONAL, 0);
	if (dma_mapping_error(&pdev->dev, iova)) {
		PCIE_ERR(dev, "PCIe: RC%d: failed to map QGIC address",
			dev->rc_idx);
		return -EIO;
	}

	msg->address_lo = iova;

	return 0;
}

static int arch_setup_msi_irq_qgic(struct pci_dev *pdev,
		struct msi_desc *desc, int nvec)
{
	int irq, index, ret, firstirq = 0;
	struct msi_msg msg;
	struct msm_pcie_dev_t *dev = PCIE_BUS_PRIV_DATA(pdev->bus);

	PCIE_DBG(dev, "RC%d\n", dev->rc_idx);

	for (index = 0; index < nvec; index++) {
		irq = msm_pcie_create_irq_qgic(dev);
		PCIE_DBG(dev, "irq %d is allocated\n", irq);

		if (irq < 0)
			return irq;

		if (index == 0)
			firstirq = irq;

		irq_set_irq_type(irq, IRQ_TYPE_EDGE_RISING);
	}

	/* write msi vector and data */
	irq_set_msi_desc(firstirq, desc);

	ret = msm_pcie_map_qgic_addr(dev, pdev, &msg);
	if (ret)
		return ret;

	msg.data = dev->msi_gicm_base + (firstirq - dev->msi[0].num);
	write_msi_msg(firstirq, &msg);

	return 0;
}

int arch_setup_msi_irq(struct pci_dev *pdev, struct msi_desc *desc)
{
	struct msm_pcie_dev_t *dev = PCIE_BUS_PRIV_DATA(pdev->bus);

	PCIE_DBG(dev, "RC%d\n", dev->rc_idx);

	if (dev->msi_gicm_addr)
		return arch_setup_msi_irq_qgic(pdev, desc, 1);
	else
		return arch_setup_msi_irq_default(pdev, desc, 1);
}

int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
{
	struct msi_desc *entry;
	int ret;
	struct msm_pcie_dev_t *pcie_dev = PCIE_BUS_PRIV_DATA(dev->bus);

	PCIE_DBG(pcie_dev, "RC%d\n", pcie_dev->rc_idx);

	if (type != PCI_CAP_ID_MSI || nvec > 32)
		return -ENOSPC;

	PCIE_DBG(pcie_dev, "nvec = %d\n", nvec);

	list_for_each_entry(entry, &dev->dev.msi_list, list) {
		entry->msi_attrib.multiple =
			__ilog2_u32(__roundup_pow_of_two(nvec));

		if (pcie_dev->msi_gicm_addr)
			ret = arch_setup_msi_irq_qgic(dev, entry, nvec);
		else
			ret = arch_setup_msi_irq_default(dev, entry, nvec);

		PCIE_DBG(pcie_dev, "ret from msi_irq: %d\n", ret);

		if (ret < 0)
			return ret;
		if (ret > 0)
			return -ENOSPC;
	}

	pcie_dev->use_msi = true;

	return 0;
}

static int msm_pcie_msi_map(struct irq_domain *domain, unsigned int irq,
	   irq_hw_number_t hwirq)
{
	irq_set_chip_and_handler (irq, &pcie_msi_chip, handle_simple_irq);
	return 0;
}

static const struct irq_domain_ops msm_pcie_msi_ops = {
	.map = msm_pcie_msi_map,
};

static int32_t msm_pcie_irq_init(struct msm_pcie_dev_t *dev)
{
	int rc;
	int msi_start =  0;
	struct device *pdev = &dev->pdev->dev;

	PCIE_DBG(dev, "RC%d\n", dev->rc_idx);

	if (dev->rc_idx)
		wakeup_source_init(&dev->ws, "RC1 pcie_wakeup_source");
	else
		wakeup_source_init(&dev->ws, "RC0 pcie_wakeup_source");

	/* register handler for linkdown interrupt */
	if (dev->irq[MSM_PCIE_INT_LINK_DOWN].num) {
		rc = devm_request_irq(pdev,
			dev->irq[MSM_PCIE_INT_LINK_DOWN].num,
			handle_linkdown_irq,
			IRQF_TRIGGER_RISING,
			dev->irq[MSM_PCIE_INT_LINK_DOWN].name,
			dev);
		if (rc) {
			PCIE_ERR(dev,
				"PCIe: Unable to request linkdown interrupt:%d\n",
				dev->irq[MSM_PCIE_INT_LINK_DOWN].num);
			return rc;
		}
	}

	/* register handler for physical MSI interrupt line */
	if (dev->irq[MSM_PCIE_INT_MSI].num) {
		rc = devm_request_irq(pdev,
			dev->irq[MSM_PCIE_INT_MSI].num,
			handle_msi_irq,
			IRQF_TRIGGER_RISING,
			dev->irq[MSM_PCIE_INT_MSI].name,
			dev);
		if (rc) {
			PCIE_ERR(dev,
				"PCIe: RC%d: Unable to request MSI interrupt\n",
				dev->rc_idx);
			return rc;
		}
	}

	/* register handler for AER interrupt */
	if (dev->irq[MSM_PCIE_INT_PLS_ERR].num) {
		rc = devm_request_irq(pdev,
				dev->irq[MSM_PCIE_INT_PLS_ERR].num,
				handle_aer_irq,
				IRQF_TRIGGER_RISING,
				dev->irq[MSM_PCIE_INT_PLS_ERR].name,
				dev);
		if (rc) {
			PCIE_ERR(dev,
				"PCIe: RC%d: Unable to request aer pls_err interrupt: %d\n",
				dev->rc_idx,
				dev->irq[MSM_PCIE_INT_PLS_ERR].num);
			return rc;
		}
	}

	/* register handler for AER legacy interrupt */
	if (dev->irq[MSM_PCIE_INT_AER_LEGACY].num) {
		rc = devm_request_irq(pdev,
				dev->irq[MSM_PCIE_INT_AER_LEGACY].num,
				handle_aer_irq,
				IRQF_TRIGGER_RISING,
				dev->irq[MSM_PCIE_INT_AER_LEGACY].name,
				dev);
		if (rc) {
			PCIE_ERR(dev,
				"PCIe: RC%d: Unable to request aer aer_legacy interrupt: %d\n",
				dev->rc_idx,
				dev->irq[MSM_PCIE_INT_AER_LEGACY].num);
			return rc;
		}
	}

	if (dev->irq[MSM_PCIE_INT_GLOBAL_INT].num) {
		rc = devm_request_irq(pdev,
				dev->irq[MSM_PCIE_INT_GLOBAL_INT].num,
				handle_global_irq,
				IRQF_TRIGGER_RISING,
				dev->irq[MSM_PCIE_INT_GLOBAL_INT].name,
				dev);
		if (rc) {
			PCIE_ERR(dev,
				"PCIe: RC%d: Unable to request global_int interrupt: %d\n",
				dev->rc_idx,
				dev->irq[MSM_PCIE_INT_GLOBAL_INT].num);
			return rc;
		}
	}

	/* register handler for PCIE_WAKE_N interrupt line */
	if (dev->wake_n) {
		rc = devm_request_irq(pdev,
				dev->wake_n, handle_wake_irq,
				IRQF_TRIGGER_FALLING, "msm_pcie_wake", dev);
		if (rc) {
			PCIE_ERR(dev,
				"PCIe: RC%d: Unable to request wake interrupt\n",
				dev->rc_idx);
			return rc;
		}

		INIT_WORK(&dev->handle_wake_work, handle_wake_func);

		rc = enable_irq_wake(dev->wake_n);
		if (rc) {
			PCIE_ERR(dev,
				"PCIe: RC%d: Unable to enable wake interrupt\n",
				dev->rc_idx);
			return rc;
		}
	}

	/* Create a virtual domain of interrupts */
	if (!dev->msi_gicm_addr) {
		dev->irq_domain = irq_domain_add_linear(dev->pdev->dev.of_node,
			PCIE_MSI_NR_IRQS, &msm_pcie_msi_ops, dev);

		if (!dev->irq_domain) {
			PCIE_ERR(dev,
				"PCIe: RC%d: Unable to initialize irq domain\n",
				dev->rc_idx);

			if (dev->wake_n)
				disable_irq(dev->wake_n);

			return PTR_ERR(dev->irq_domain);
		}

		msi_start = irq_create_mapping(dev->irq_domain, 0);
	}

	return 0;
}

static void msm_pcie_irq_deinit(struct msm_pcie_dev_t *dev)
{
	PCIE_DBG(dev, "RC%d\n", dev->rc_idx);

	wakeup_source_trash(&dev->ws);

	if (dev->wake_n)
		disable_irq(dev->wake_n);
}

static void msm_pcie_config_l0s(struct msm_pcie_dev_t *dev,
				struct pci_dev *pdev, bool enable)
{
	u32 val;
	u32 lnkcap_offset = pdev->pcie_cap + PCI_EXP_LNKCAP;
	u32 lnkctl_offset = pdev->pcie_cap + PCI_EXP_LNKCTL;

	pci_read_config_dword(pdev, lnkcap_offset, &val);
	if (!(val & BIT(10))) {
		PCIE_DBG(dev,
			"PCIe: RC%d: PCI device does not support L0s\n",
			dev->rc_idx);
		return;
	}

	if (enable)
		msm_pcie_config_clear_set_dword(pdev, lnkctl_offset, 0,
			PCI_EXP_LNKCTL_ASPM_L0S);
	else
		msm_pcie_config_clear_set_dword(pdev, lnkctl_offset,
			PCI_EXP_LNKCTL_ASPM_L0S, 0);

	pci_read_config_dword(pdev, lnkctl_offset, &val);
	PCIE_DBG2(dev, "PCIe: RC%d: LINKCTRLSTATUS:0x%x\n", dev->rc_idx, val);
}

static int msm_pcie_config_l0s_disable(struct pci_dev *pdev, void *dev)
{
	struct msm_pcie_dev_t *pcie_dev = (struct msm_pcie_dev_t *)dev;

	msm_pcie_config_l0s(pcie_dev, pdev, false);
	return 0;
}

static int msm_pcie_config_l0s_enable(struct pci_dev *pdev, void *dev)
{
	struct msm_pcie_dev_t *pcie_dev = (struct msm_pcie_dev_t *)dev;

	msm_pcie_config_l0s(pcie_dev, pdev, true);
	return 0;
}

static void msm_pcie_config_l1(struct msm_pcie_dev_t *dev,
				struct pci_dev *pdev, bool enable)
{
	u32 val;
	u32 lnkcap_offset = pdev->pcie_cap + PCI_EXP_LNKCAP;
	u32 lnkctl_offset = pdev->pcie_cap + PCI_EXP_LNKCTL;

	pci_read_config_dword(pdev, lnkcap_offset, &val);
	if (!(val & BIT(11))) {
		PCIE_DBG(dev,
			"PCIe: RC%d: PCI device does not support L1\n",
			dev->rc_idx);
		return;
	}

	if (enable)
		msm_pcie_config_clear_set_dword(pdev, lnkctl_offset, 0,
			PCI_EXP_LNKCTL_ASPM_L1);
	else
		msm_pcie_config_clear_set_dword(pdev, lnkctl_offset,
			PCI_EXP_LNKCTL_ASPM_L1, 0);

	pci_read_config_dword(pdev, lnkctl_offset, &val);
	PCIE_DBG2(dev, "PCIe: RC%d: LINKCTRLSTATUS:0x%x\n", dev->rc_idx, val);
}

static int msm_pcie_config_l1_disable(struct pci_dev *pdev, void *dev)
{
	struct msm_pcie_dev_t *pcie_dev = (struct msm_pcie_dev_t *)dev;

	msm_pcie_config_l1(pcie_dev, pdev, false);
	return 0;
}

static int msm_pcie_config_l1_enable(struct pci_dev *pdev, void *dev)
{
	struct msm_pcie_dev_t *pcie_dev = (struct msm_pcie_dev_t *)dev;

	msm_pcie_config_l1(pcie_dev, pdev, true);
	return 0;
}

static void msm_pcie_config_l1ss(struct msm_pcie_dev_t *dev,
				struct pci_dev *pdev, bool enable)
{
	bool l1_1_pcipm_support, l1_2_pcipm_support;
	bool l1_1_aspm_support, l1_2_aspm_support;
	u32 val, val2;
	u32 l1ss_cap_id_offset, l1ss_cap_offset, l1ss_ctl1_offset;
	u32 devctl2_offset = pdev->pcie_cap + PCI_EXP_DEVCTL2;

	l1ss_cap_id_offset = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS);
	if (!l1ss_cap_id_offset) {
		PCIE_DBG(dev,
			"PCIe: RC%d could not find L1ss capability register for device\n",
			dev->rc_idx);
		return;
	}

	l1ss_cap_offset = l1ss_cap_id_offset + PCI_L1SS_CAP;
	l1ss_ctl1_offset = l1ss_cap_id_offset + PCI_L1SS_CTL1;

	pci_read_config_dword(pdev, l1ss_cap_offset, &val);
	l1_1_pcipm_support = !!(val & (PCI_L1SS_CAP_PCIPM_L1_1));
	l1_2_pcipm_support = !!(val & (PCI_L1SS_CAP_PCIPM_L1_2));
	l1_1_aspm_support = !!(val & (PCI_L1SS_CAP_ASPM_L1_1));
	l1_2_aspm_support = !!(val & (PCI_L1SS_CAP_ASPM_L1_2));
	if (!l1_1_pcipm_support && !l1_2_pcipm_support &&
		!l1_1_aspm_support && !l1_2_aspm_support) {
		PCIE_DBG(dev,
			"PCIe: RC%d: PCI device does not support any L1ss\n",
			dev->rc_idx);
		return;
	}

	/* Enable the AUX Clock and the Core Clk to be synchronous for L1ss */
	if (pci_is_root_bus(pdev->bus) && !dev->aux_clk_sync) {
		if (enable)
			msm_pcie_write_mask(dev->parf +
				PCIE20_PARF_SYS_CTRL, BIT(3), 0);
		else
			msm_pcie_write_mask(dev->parf +
				PCIE20_PARF_SYS_CTRL, 0, BIT(3));
	}

	if (enable) {
		msm_pcie_config_clear_set_dword(pdev, devctl2_offset, 0,
			PCI_EXP_DEVCTL2_LTR_EN);
		msm_pcie_config_clear_set_dword(pdev, l1ss_ctl1_offset, 0,
			(l1_1_pcipm_support ? PCI_L1SS_CTL1_PCIPM_L1_1 : 0) |
			(l1_2_pcipm_support ? PCI_L1SS_CTL1_PCIPM_L1_2 : 0) |
			(l1_1_aspm_support ? PCI_L1SS_CTL1_ASPM_L1_1 : 0) |
			(l1_2_aspm_support ? PCI_L1SS_CTL1_ASPM_L1_2 : 0));
	} else {
		msm_pcie_config_clear_set_dword(pdev, devctl2_offset,
			PCI_EXP_DEVCTL2_LTR_EN, 0);
		msm_pcie_config_clear_set_dword(pdev, l1ss_ctl1_offset,
			(l1_1_pcipm_support ? PCI_L1SS_CTL1_PCIPM_L1_1 : 0) |
			(l1_2_pcipm_support ? PCI_L1SS_CTL1_PCIPM_L1_2 : 0) |
			(l1_1_aspm_support ? PCI_L1SS_CTL1_ASPM_L1_1 : 0) |
			(l1_2_aspm_support ? PCI_L1SS_CTL1_ASPM_L1_2 : 0), 0);
	}

	pci_read_config_dword(pdev, l1ss_ctl1_offset, &val);
	PCIE_DBG2(dev, "PCIe: RC%d: L1SUB_CONTROL1:0x%x\n", dev->rc_idx, val);

	pci_read_config_dword(pdev, devctl2_offset, &val2);
	PCIE_DBG2(dev, "PCIe: RC%d: DEVICE_CONTROL2_STATUS2::0x%x\n",
		dev->rc_idx, val2);
}

static int msm_pcie_config_l1ss_disable(struct pci_dev *pdev, void *dev)
{
	struct msm_pcie_dev_t *pcie_dev = (struct msm_pcie_dev_t *)dev;

	msm_pcie_config_l1ss(pcie_dev, pdev, false);
	return 0;
}

static int msm_pcie_config_l1ss_enable(struct pci_dev *pdev, void *dev)
{
	struct msm_pcie_dev_t *pcie_dev = (struct msm_pcie_dev_t *)dev;

	msm_pcie_config_l1ss(pcie_dev, pdev, true);
	return 0;
}

static void msm_pcie_config_clock_power_management(struct msm_pcie_dev_t *dev,
				struct pci_dev *pdev)
{
	u32 val;
	u32 lnkcap_offset = pdev->pcie_cap + PCI_EXP_LNKCAP;
	u32 lnkctl_offset = pdev->pcie_cap + PCI_EXP_LNKCTL;

	if (pci_is_root_bus(pdev->bus))
		return;

	pci_read_config_dword(pdev, lnkcap_offset, &val);
	if (val & PCI_EXP_LNKCAP_CLKPM)
		msm_pcie_config_clear_set_dword(pdev, lnkctl_offset, 0,
			PCI_EXP_LNKCTL_CLKREQ_EN);
	else
		PCIE_DBG(dev,
			"PCIe: RC%d: PCI device does not support clock power management\n",
			dev->rc_idx);
}

static void msm_pcie_config_link_pm(struct msm_pcie_dev_t *dev,
				struct pci_dev *pdev, bool enable)
{
	if (dev->common_clk_en)
		msm_pcie_config_clear_set_dword(pdev,
			pdev->pcie_cap + PCI_EXP_LNKCTL, 0,
			PCI_EXP_LNKCTL_CCC);

	if (dev->clk_power_manage_en)
		msm_pcie_config_clock_power_management(dev, pdev);
	if (dev->l0s_supported)
		msm_pcie_config_l0s(dev, pdev, enable);
	if (dev->l1ss_supported)
		msm_pcie_config_l1ss(dev, pdev, enable);
	if (dev->l1_supported)
		msm_pcie_config_l1(dev, pdev, enable);
}

static void msm_pcie_config_link_pm_rc(struct msm_pcie_dev_t *dev,
				struct pci_dev *pdev, bool enable)
{
	bool child_l0s_enable = 0, child_l1_enable = 0, child_l1ss_enable = 0;

	if (!pdev->subordinate || !(&pdev->subordinate->devices)) {
		PCIE_DBG(dev,
			"PCIe: RC%d: no device connected to root complex\n",
			dev->rc_idx);
		return;
	}

	if (dev->l0s_supported) {
		struct pci_dev *child_pdev, *c_pdev;

		list_for_each_entry_safe(child_pdev, c_pdev,
			&pdev->subordinate->devices, bus_list) {
			u32 val;

			pci_read_config_dword(child_pdev,
				child_pdev->pcie_cap + PCI_EXP_LNKCTL, &val);
			child_l0s_enable = !!(val & PCI_EXP_LNKCTL_ASPM_L0S);
			if (child_l0s_enable)
				break;
		}

		if (child_l0s_enable)
			msm_pcie_config_l0s(dev, pdev, enable);
		else
			dev->l0s_supported = false;
	}

	if (dev->l1ss_supported) {
		struct pci_dev *child_pdev, *c_pdev;

		list_for_each_entry_safe(child_pdev, c_pdev,
			&pdev->subordinate->devices, bus_list) {
			u32 val;
			u32 l1ss_cap_id_offset =
				pci_find_ext_capability(child_pdev,
					PCI_EXT_CAP_ID_L1SS);

			if (!l1ss_cap_id_offset)
				continue;

			pci_read_config_dword(child_pdev,
				l1ss_cap_id_offset + PCI_L1SS_CTL1, &val);
			child_l1ss_enable = !!(val &
				(PCI_L1SS_CTL1_PCIPM_L1_1 |
				PCI_L1SS_CTL1_PCIPM_L1_2 |
				PCI_L1SS_CTL1_ASPM_L1_1 |
				PCI_L1SS_CTL1_ASPM_L1_2));
			if (child_l1ss_enable)
				break;
		}

		if (child_l1ss_enable)
			msm_pcie_config_l1ss(dev, pdev, enable);
		else
			dev->l1ss_supported = false;
	}

	if (dev->l1_supported) {
		struct pci_dev *child_pdev, *c_pdev;

		list_for_each_entry_safe(child_pdev, c_pdev,
			&pdev->subordinate->devices, bus_list) {
			u32 val;

			pci_read_config_dword(child_pdev,
				child_pdev->pcie_cap + PCI_EXP_LNKCTL, &val);
			child_l1_enable = !!(val & PCI_EXP_LNKCTL_ASPM_L1);
			if (child_l1_enable)
				break;
		}

		if (child_l1_enable)
			msm_pcie_config_l1(dev, pdev, enable);
		else
			dev->l1_supported = false;
	}
}

static int msm_pcie_config_device(struct pci_dev *dev, void *pdev)
{
	struct msm_pcie_dev_t *pcie_dev = (struct msm_pcie_dev_t *)pdev;
	u8 busnr = dev->bus->number;
	u8 slot = PCI_SLOT(dev->devfn);
	u8 func = PCI_FUNC(dev->devfn);

	PCIE_DBG(pcie_dev, "PCIe: RC%d: configure PCI device %02x:%02x.%01x\n",
		pcie_dev->rc_idx, busnr, slot, func);

	msm_pcie_configure_sid(pcie_dev, dev);

	if (!pci_is_root_bus(dev->bus))
		msm_pcie_config_link_pm(pcie_dev, dev, true);

	return 0;
}

/* Hook to setup PCI device during PCI framework scan */
int pcibios_add_device(struct pci_dev *dev)
{
	struct msm_pcie_dev_t *pcie_dev = PCIE_BUS_PRIV_DATA(dev->bus);

	return msm_pcie_config_device(dev, pcie_dev);
}

static int msm_pcie_probe(struct platform_device *pdev)
{
	int ret = 0;
	int rc_idx = -1;
	int i, j;

	PCIE_GEN_DBG("%s\n", __func__);

	mutex_lock(&pcie_drv.drv_lock);

	ret = of_property_read_u32((&pdev->dev)->of_node,
				"cell-index", &rc_idx);
	if (ret) {
		PCIE_GEN_DBG("Did not find RC index.\n");
		goto out;
	} else {
		if (rc_idx >= MAX_RC_NUM) {
			pr_err(
				"PCIe: Invalid RC Index %d (max supported = %d)\n",
				rc_idx, MAX_RC_NUM);
			goto out;
		}
		pcie_drv.rc_num++;
		PCIE_DBG(&msm_pcie_dev[rc_idx], "PCIe: RC index is %d.\n",
			rc_idx);
	}

	msm_pcie_dev[rc_idx].l0s_supported =
		of_property_read_bool((&pdev->dev)->of_node,
				"qcom,l0s-supported");
	if (msm_pcie_invert_l0s_support & BIT(rc_idx))
		msm_pcie_dev[rc_idx].l0s_supported =
			!msm_pcie_dev[rc_idx].l0s_supported;
	PCIE_DBG(&msm_pcie_dev[rc_idx], "L0s is %s supported.\n",
		msm_pcie_dev[rc_idx].l0s_supported ? "" : "not");
	msm_pcie_dev[rc_idx].l1_supported =
		of_property_read_bool((&pdev->dev)->of_node,
				"qcom,l1-supported");
	if (msm_pcie_invert_l1_support & BIT(rc_idx))
		msm_pcie_dev[rc_idx].l1_supported =
			!msm_pcie_dev[rc_idx].l1_supported;
	PCIE_DBG(&msm_pcie_dev[rc_idx], "L1 is %s supported.\n",
		msm_pcie_dev[rc_idx].l1_supported ? "" : "not");
	msm_pcie_dev[rc_idx].l1ss_supported =
		of_property_read_bool((&pdev->dev)->of_node,
				"qcom,l1ss-supported");
	if (msm_pcie_invert_l1ss_support & BIT(rc_idx))
		msm_pcie_dev[rc_idx].l1ss_supported =
			!msm_pcie_dev[rc_idx].l1ss_supported;
	PCIE_DBG(&msm_pcie_dev[rc_idx], "L1ss is %s supported.\n",
		msm_pcie_dev[rc_idx].l1ss_supported ? "" : "not");
	msm_pcie_dev[rc_idx].common_clk_en =
		of_property_read_bool((&pdev->dev)->of_node,
				"qcom,common-clk-en");
	PCIE_DBG(&msm_pcie_dev[rc_idx], "Common clock is %s enabled.\n",
		msm_pcie_dev[rc_idx].common_clk_en ? "" : "not");
	msm_pcie_dev[rc_idx].clk_power_manage_en =
		of_property_read_bool((&pdev->dev)->of_node,
				"qcom,clk-power-manage-en");
	PCIE_DBG(&msm_pcie_dev[rc_idx],
		"Clock power management is %s enabled.\n",
		msm_pcie_dev[rc_idx].clk_power_manage_en ? "" : "not");
	msm_pcie_dev[rc_idx].aux_clk_sync =
		of_property_read_bool((&pdev->dev)->of_node,
				"qcom,aux-clk-sync");
	PCIE_DBG(&msm_pcie_dev[rc_idx],
		"AUX clock is %s synchronous to Core clock.\n",
		msm_pcie_dev[rc_idx].aux_clk_sync ? "" : "not");

	msm_pcie_dev[rc_idx].use_19p2mhz_aux_clk =
		of_property_read_bool((&pdev->dev)->of_node,
				"qcom,use-19p2mhz-aux-clk");
	PCIE_DBG(&msm_pcie_dev[rc_idx],
		"AUX clock frequency is %s 19.2MHz.\n",
		msm_pcie_dev[rc_idx].use_19p2mhz_aux_clk ? "" : "not");

	msm_pcie_dev[rc_idx].smmu_exist =
		of_property_read_bool((&pdev->dev)->of_node,
				"qcom,smmu-exist");
	PCIE_DBG(&msm_pcie_dev[rc_idx],
		"SMMU does %s exist.\n",
		msm_pcie_dev[rc_idx].smmu_exist ? "" : "not");

	msm_pcie_dev[rc_idx].smmu_sid_base = 0;
	ret = of_property_read_u32((&pdev->dev)->of_node, "qcom,smmu-sid-base",
				&msm_pcie_dev[rc_idx].smmu_sid_base);
	if (ret)
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"RC%d SMMU sid base not found\n",
			msm_pcie_dev[rc_idx].rc_idx);
	else
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"RC%d: qcom,smmu-sid-base: 0x%x.\n",
			msm_pcie_dev[rc_idx].rc_idx,
			msm_pcie_dev[rc_idx].smmu_sid_base);

	msm_pcie_dev[rc_idx].boot_option = 0;
	ret = of_property_read_u32((&pdev->dev)->of_node, "qcom,boot-option",
				&msm_pcie_dev[rc_idx].boot_option);
	PCIE_DBG(&msm_pcie_dev[rc_idx],
		"PCIe: RC%d boot option is 0x%x.\n",
		rc_idx, msm_pcie_dev[rc_idx].boot_option);

	msm_pcie_dev[rc_idx].phy_ver = 1;
	ret = of_property_read_u32((&pdev->dev)->of_node,
				"qcom,pcie-phy-ver",
				&msm_pcie_dev[rc_idx].phy_ver);
	if (ret)
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"RC%d: pcie-phy-ver does not exist.\n",
			msm_pcie_dev[rc_idx].rc_idx);
	else
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"RC%d: pcie-phy-ver: %d.\n",
			msm_pcie_dev[rc_idx].rc_idx,
			msm_pcie_dev[rc_idx].phy_ver);

	msm_pcie_dev[rc_idx].n_fts = 0;
	ret = of_property_read_u32((&pdev->dev)->of_node,
				"qcom,n-fts",
				&msm_pcie_dev[rc_idx].n_fts);

	if (ret)
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"n-fts does not exist. ret=%d\n", ret);
	else
		PCIE_DBG(&msm_pcie_dev[rc_idx], "n-fts: 0x%x.\n",
				msm_pcie_dev[rc_idx].n_fts);

	msm_pcie_dev[rc_idx].max_link_speed = GEN2_SPEED;
	ret = of_property_read_u32(pdev->dev.of_node,
				"qcom,max-link-speed",
				&msm_pcie_dev[rc_idx].max_link_speed);
	PCIE_DBG(&msm_pcie_dev[rc_idx], "PCIe: RC%d: max-link-speed: 0x%x.\n",
		rc_idx, msm_pcie_dev[rc_idx].max_link_speed);

	msm_pcie_dev[rc_idx].ext_ref_clk =
		of_property_read_bool((&pdev->dev)->of_node,
				"qcom,ext-ref-clk");
	PCIE_DBG(&msm_pcie_dev[rc_idx], "ref clk is %s.\n",
		msm_pcie_dev[rc_idx].ext_ref_clk ? "external" : "internal");

	msm_pcie_dev[rc_idx].ep_latency = 0;
	ret = of_property_read_u32((&pdev->dev)->of_node,
				"qcom,ep-latency",
				&msm_pcie_dev[rc_idx].ep_latency);
	if (ret)
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"RC%d: ep-latency does not exist.\n",
			rc_idx);
	else
		PCIE_DBG(&msm_pcie_dev[rc_idx], "RC%d: ep-latency: 0x%x.\n",
			rc_idx, msm_pcie_dev[rc_idx].ep_latency);

	msm_pcie_dev[rc_idx].switch_latency = 0;
	ret = of_property_read_u32((&pdev->dev)->of_node,
					"qcom,switch-latency",
					&msm_pcie_dev[rc_idx].switch_latency);

	if (ret)
		PCIE_DBG(&msm_pcie_dev[rc_idx],
				"RC%d: switch-latency does not exist.\n",
				rc_idx);
	else
		PCIE_DBG(&msm_pcie_dev[rc_idx],
				"RC%d: switch-latency: 0x%x.\n",
				rc_idx, msm_pcie_dev[rc_idx].switch_latency);

	msm_pcie_dev[rc_idx].wr_halt_size = 0;
	ret = of_property_read_u32(pdev->dev.of_node,
				"qcom,wr-halt-size",
				&msm_pcie_dev[rc_idx].wr_halt_size);
	if (ret)
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"RC%d: wr-halt-size not specified in dt. Use default value.\n",
			rc_idx);
	else
		PCIE_DBG(&msm_pcie_dev[rc_idx], "RC%d: wr-halt-size: 0x%x.\n",
			rc_idx, msm_pcie_dev[rc_idx].wr_halt_size);

	msm_pcie_dev[rc_idx].slv_addr_space_size = SZ_16M;
	ret = of_property_read_u32(pdev->dev.of_node,
				"qcom,slv-addr-space-size",
				&msm_pcie_dev[rc_idx].slv_addr_space_size);
	PCIE_DBG(&msm_pcie_dev[rc_idx],
		"RC%d: slv-addr-space-size: 0x%x.\n",
		rc_idx, msm_pcie_dev[rc_idx].slv_addr_space_size);

	msm_pcie_dev[rc_idx].phy_status_offset = 0;
	ret = of_property_read_u32(pdev->dev.of_node,
				"qcom,phy-status-offset",
				&msm_pcie_dev[rc_idx].phy_status_offset);
	if (ret) {
		PCIE_ERR(&msm_pcie_dev[rc_idx],
			"RC%d: failed to get PCIe PHY status offset.\n",
			rc_idx);
		goto decrease_rc_num;
	} else {
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"RC%d: phy-status-offset: 0x%x.\n",
			rc_idx, msm_pcie_dev[rc_idx].phy_status_offset);
	}

	msm_pcie_dev[rc_idx].cpl_timeout = 0;
	ret = of_property_read_u32((&pdev->dev)->of_node,
				"qcom,cpl-timeout",
				&msm_pcie_dev[rc_idx].cpl_timeout);
	if (ret)
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"RC%d: Using default cpl-timeout.\n",
			rc_idx);
	else
		PCIE_DBG(&msm_pcie_dev[rc_idx], "RC%d: cpl-timeout: 0x%x.\n",
			rc_idx, msm_pcie_dev[rc_idx].cpl_timeout);

	msm_pcie_dev[rc_idx].perst_delay_us_min =
		PERST_PROPAGATION_DELAY_US_MIN;
	ret = of_property_read_u32(pdev->dev.of_node,
				"qcom,perst-delay-us-min",
				&msm_pcie_dev[rc_idx].perst_delay_us_min);
	if (ret)
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"RC%d: perst-delay-us-min does not exist. Use default value %dus.\n",
			rc_idx, msm_pcie_dev[rc_idx].perst_delay_us_min);
	else
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"RC%d: perst-delay-us-min: %dus.\n",
			rc_idx, msm_pcie_dev[rc_idx].perst_delay_us_min);

	msm_pcie_dev[rc_idx].perst_delay_us_max =
		PERST_PROPAGATION_DELAY_US_MAX;
	ret = of_property_read_u32(pdev->dev.of_node,
				"qcom,perst-delay-us-max",
				&msm_pcie_dev[rc_idx].perst_delay_us_max);
	if (ret)
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"RC%d: perst-delay-us-max does not exist. Use default value %dus.\n",
			rc_idx, msm_pcie_dev[rc_idx].perst_delay_us_max);
	else
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"RC%d: perst-delay-us-max: %dus.\n",
			rc_idx, msm_pcie_dev[rc_idx].perst_delay_us_max);

	msm_pcie_dev[rc_idx].tlp_rd_size = PCIE_TLP_RD_SIZE;
	ret = of_property_read_u32(pdev->dev.of_node,
				"qcom,tlp-rd-size",
				&msm_pcie_dev[rc_idx].tlp_rd_size);
	if (ret)
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"RC%d: tlp-rd-size does not exist. tlp-rd-size: 0x%x.\n",
			rc_idx, msm_pcie_dev[rc_idx].tlp_rd_size);
	else
		PCIE_DBG(&msm_pcie_dev[rc_idx], "RC%d: tlp-rd-size: 0x%x.\n",
			rc_idx, msm_pcie_dev[rc_idx].tlp_rd_size);

	msm_pcie_dev[rc_idx].msi_gicm_addr = 0;
	msm_pcie_dev[rc_idx].msi_gicm_base = 0;
	ret = of_property_read_u32((&pdev->dev)->of_node,
				"qcom,msi-gicm-addr",
				&msm_pcie_dev[rc_idx].msi_gicm_addr);

	if (ret) {
		PCIE_DBG(&msm_pcie_dev[rc_idx], "%s",
			"msi-gicm-addr does not exist.\n");
	} else {
		PCIE_DBG(&msm_pcie_dev[rc_idx], "msi-gicm-addr: 0x%x.\n",
				msm_pcie_dev[rc_idx].msi_gicm_addr);

		ret = of_property_read_u32((&pdev->dev)->of_node,
				"qcom,msi-gicm-base",
				&msm_pcie_dev[rc_idx].msi_gicm_base);

		if (ret) {
			PCIE_ERR(&msm_pcie_dev[rc_idx],
				"PCIe: RC%d: msi-gicm-base does not exist.\n",
				rc_idx);
			goto decrease_rc_num;
		} else {
			PCIE_DBG(&msm_pcie_dev[rc_idx], "msi-gicm-base: 0x%x\n",
					msm_pcie_dev[rc_idx].msi_gicm_base);
		}
	}

	msm_pcie_dev[rc_idx].scm_dev_id = 0;
	ret = of_property_read_u32((&pdev->dev)->of_node,
				"qcom,scm-dev-id",
				&msm_pcie_dev[rc_idx].scm_dev_id);

	msm_pcie_dev[rc_idx].rc_idx = rc_idx;
	msm_pcie_dev[rc_idx].pdev = pdev;
	msm_pcie_dev[rc_idx].vreg_n = 0;
	msm_pcie_dev[rc_idx].gpio_n = 0;
	msm_pcie_dev[rc_idx].parf_deemph = 0;
	msm_pcie_dev[rc_idx].parf_swing = 0;
	msm_pcie_dev[rc_idx].link_status = MSM_PCIE_LINK_DEINIT;
	msm_pcie_dev[rc_idx].user_suspend = false;
	msm_pcie_dev[rc_idx].disable_pc = false;
	msm_pcie_dev[rc_idx].saved_state = NULL;
	msm_pcie_dev[rc_idx].enumerated = false;
	msm_pcie_dev[rc_idx].num_active_ep = 0;
	msm_pcie_dev[rc_idx].num_ep = 0;
	msm_pcie_dev[rc_idx].pending_ep_reg = false;
	msm_pcie_dev[rc_idx].phy_len = 0;
	msm_pcie_dev[rc_idx].phy_sequence = NULL;
	msm_pcie_dev[rc_idx].event_reg = NULL;
	msm_pcie_dev[rc_idx].linkdown_counter = 0;
	msm_pcie_dev[rc_idx].link_turned_on_counter = 0;
	msm_pcie_dev[rc_idx].link_turned_off_counter = 0;
	msm_pcie_dev[rc_idx].rc_corr_counter = 0;
	msm_pcie_dev[rc_idx].rc_non_fatal_counter = 0;
	msm_pcie_dev[rc_idx].rc_fatal_counter = 0;
	msm_pcie_dev[rc_idx].ep_corr_counter = 0;
	msm_pcie_dev[rc_idx].ep_non_fatal_counter = 0;
	msm_pcie_dev[rc_idx].ep_fatal_counter = 0;
	msm_pcie_dev[rc_idx].suspending = false;
	msm_pcie_dev[rc_idx].wake_counter = 0;
	msm_pcie_dev[rc_idx].aer_enable = true;
	if (msm_pcie_invert_aer_support)
		msm_pcie_dev[rc_idx].aer_enable = false;
	msm_pcie_dev[rc_idx].power_on = false;
	msm_pcie_dev[rc_idx].use_msi = false;
	msm_pcie_dev[rc_idx].use_pinctrl = false;
	msm_pcie_dev[rc_idx].linkdown_panic = false;
	msm_pcie_dev[rc_idx].bridge_found = false;
	memcpy(msm_pcie_dev[rc_idx].vreg, msm_pcie_vreg_info,
				sizeof(msm_pcie_vreg_info));
	memcpy(msm_pcie_dev[rc_idx].gpio, msm_pcie_gpio_info,
				sizeof(msm_pcie_gpio_info));
	memcpy(msm_pcie_dev[rc_idx].clk, msm_pcie_clk_info[rc_idx],
				sizeof(msm_pcie_clk_info[rc_idx]));
	memcpy(msm_pcie_dev[rc_idx].pipeclk, msm_pcie_pipe_clk_info[rc_idx],
				sizeof(msm_pcie_pipe_clk_info[rc_idx]));
	memcpy(msm_pcie_dev[rc_idx].res, msm_pcie_res_info,
				sizeof(msm_pcie_res_info));
	memcpy(msm_pcie_dev[rc_idx].irq, msm_pcie_irq_info,
				sizeof(msm_pcie_irq_info));
	memcpy(msm_pcie_dev[rc_idx].msi, msm_pcie_msi_info,
				sizeof(msm_pcie_msi_info));
	memcpy(msm_pcie_dev[rc_idx].reset, msm_pcie_reset_info[rc_idx],
				sizeof(msm_pcie_reset_info[rc_idx]));
	memcpy(msm_pcie_dev[rc_idx].pipe_reset,
			msm_pcie_pipe_reset_info[rc_idx],
			sizeof(msm_pcie_pipe_reset_info[rc_idx]));
	msm_pcie_dev[rc_idx].shadow_en = true;
	for (i = 0; i < PCIE_CONF_SPACE_DW; i++)
		msm_pcie_dev[rc_idx].rc_shadow[i] = PCIE_CLEAR;
	for (i = 0; i < MAX_DEVICE_NUM; i++)
		for (j = 0; j < PCIE_CONF_SPACE_DW; j++)
			msm_pcie_dev[rc_idx].ep_shadow[i][j] = PCIE_CLEAR;
	for (i = 0; i < MAX_DEVICE_NUM; i++) {
		msm_pcie_dev[rc_idx].pcidev_table[i].bdf = 0;
		msm_pcie_dev[rc_idx].pcidev_table[i].dev = NULL;
		msm_pcie_dev[rc_idx].pcidev_table[i].short_bdf = 0;
		msm_pcie_dev[rc_idx].pcidev_table[i].sid = 0;
		msm_pcie_dev[rc_idx].pcidev_table[i].domain = rc_idx;
		msm_pcie_dev[rc_idx].pcidev_table[i].conf_base = NULL;
		msm_pcie_dev[rc_idx].pcidev_table[i].phy_address = 0;
		msm_pcie_dev[rc_idx].pcidev_table[i].dev_ctrlstts_offset = 0;
		msm_pcie_dev[rc_idx].pcidev_table[i].event_reg = NULL;
		msm_pcie_dev[rc_idx].pcidev_table[i].registered = true;
	}

	dev_set_drvdata(&msm_pcie_dev[rc_idx].pdev->dev, &msm_pcie_dev[rc_idx]);

	ret = msm_pcie_get_resources(&msm_pcie_dev[rc_idx],
				msm_pcie_dev[rc_idx].pdev);

	if (ret)
		goto decrease_rc_num;

	msm_pcie_dev[rc_idx].pinctrl = devm_pinctrl_get(&pdev->dev);
	if (IS_ERR_OR_NULL(msm_pcie_dev[rc_idx].pinctrl))
		PCIE_ERR(&msm_pcie_dev[rc_idx],
			"PCIe: RC%d failed to get pinctrl\n",
			rc_idx);
	else
		msm_pcie_dev[rc_idx].use_pinctrl = true;

	if (msm_pcie_dev[rc_idx].use_pinctrl) {
		msm_pcie_dev[rc_idx].pins_default =
			pinctrl_lookup_state(msm_pcie_dev[rc_idx].pinctrl,
						"default");
		if (IS_ERR(msm_pcie_dev[rc_idx].pins_default)) {
			PCIE_ERR(&msm_pcie_dev[rc_idx],
				"PCIe: RC%d could not get pinctrl default state\n",
				rc_idx);
			msm_pcie_dev[rc_idx].pins_default = NULL;
		}

		msm_pcie_dev[rc_idx].pins_sleep =
			pinctrl_lookup_state(msm_pcie_dev[rc_idx].pinctrl,
						"sleep");
		if (IS_ERR(msm_pcie_dev[rc_idx].pins_sleep)) {
			PCIE_ERR(&msm_pcie_dev[rc_idx],
				"PCIe: RC%d could not get pinctrl sleep state\n",
				rc_idx);
			msm_pcie_dev[rc_idx].pins_sleep = NULL;
		}
	}

	ret = msm_pcie_gpio_init(&msm_pcie_dev[rc_idx]);
	if (ret) {
		msm_pcie_release_resources(&msm_pcie_dev[rc_idx]);
		goto decrease_rc_num;
	}

	ret = msm_pcie_irq_init(&msm_pcie_dev[rc_idx]);
	if (ret) {
		msm_pcie_release_resources(&msm_pcie_dev[rc_idx]);
		msm_pcie_gpio_deinit(&msm_pcie_dev[rc_idx]);
		goto decrease_rc_num;
	}

	msm_pcie_sysfs_init(&msm_pcie_dev[rc_idx]);

	msm_pcie_dev[rc_idx].drv_ready = true;

	if (msm_pcie_dev[rc_idx].boot_option &
			MSM_PCIE_NO_PROBE_ENUMERATION) {
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"PCIe: RC%d will be enumerated by client or endpoint.\n",
			rc_idx);
		mutex_unlock(&pcie_drv.drv_lock);
		return 0;
	}

	ret = msm_pcie_enumerate(rc_idx);

	if (ret)
		PCIE_ERR(&msm_pcie_dev[rc_idx],
			"PCIe: RC%d is not enabled during bootup; it will be enumerated upon client request.\n",
			rc_idx);
	else
		PCIE_ERR(&msm_pcie_dev[rc_idx], "RC%d is enabled in bootup\n",
			rc_idx);

	PCIE_DBG(&msm_pcie_dev[rc_idx], "PCIE probed %s\n",
		dev_name(&(pdev->dev)));

	mutex_unlock(&pcie_drv.drv_lock);
	return 0;

decrease_rc_num:
	pcie_drv.rc_num--;
out:
	if (rc_idx < 0 || rc_idx >= MAX_RC_NUM)
		pr_err("PCIe: Invalid RC index %d. Driver probe failed\n",
		rc_idx);
	else
		PCIE_ERR(&msm_pcie_dev[rc_idx],
			"PCIe: Driver probe failed for RC%d:%d\n",
			rc_idx, ret);

	mutex_unlock(&pcie_drv.drv_lock);

	return ret;
}

static int msm_pcie_remove(struct platform_device *pdev)
{
	int ret = 0;
	int rc_idx;

	PCIE_GEN_DBG("PCIe:%s.\n", __func__);

	mutex_lock(&pcie_drv.drv_lock);

	ret = of_property_read_u32((&pdev->dev)->of_node,
				"cell-index", &rc_idx);
	if (ret) {
		pr_err("%s: Did not find RC index.\n", __func__);
		goto out;
	} else {
		pcie_drv.rc_num--;
		PCIE_GEN_DBG("%s: RC index is 0x%x.", __func__, rc_idx);
	}

	msm_pcie_irq_deinit(&msm_pcie_dev[rc_idx]);
	msm_pcie_vreg_deinit(&msm_pcie_dev[rc_idx]);
	msm_pcie_clk_deinit(&msm_pcie_dev[rc_idx]);
	msm_pcie_gpio_deinit(&msm_pcie_dev[rc_idx]);
	msm_pcie_release_resources(&msm_pcie_dev[rc_idx]);

out:
	mutex_unlock(&pcie_drv.drv_lock);

	return ret;
}

static const struct of_device_id msm_pcie_match[] = {
	{	.compatible = "qcom,pci-msm",
	},
	{}
};

static struct platform_driver msm_pcie_driver = {
	.probe	= msm_pcie_probe,
	.remove	= msm_pcie_remove,
	.driver	= {
		.name		= "pci-msm",
		.owner		= THIS_MODULE,
		.of_match_table	= msm_pcie_match,
	},
};

static int __init pcie_init(void)
{
	int ret = 0, i;
	char rc_name[MAX_RC_NAME_LEN];

	pr_alert("pcie:%s.\n", __func__);

	pcie_drv.rc_num = 0;
	mutex_init(&pcie_drv.drv_lock);

	for (i = 0; i < MAX_RC_NUM; i++) {
		snprintf(rc_name, MAX_RC_NAME_LEN, "pcie%d-short", i);
		msm_pcie_dev[i].ipc_log =
			ipc_log_context_create(PCIE_LOG_PAGES, rc_name, 0);
		if (msm_pcie_dev[i].ipc_log == NULL)
			pr_err("%s: unable to create IPC log context for %s\n",
				__func__, rc_name);
		else
			PCIE_DBG(&msm_pcie_dev[i],
				"PCIe IPC logging is enable for RC%d\n",
				i);
		snprintf(rc_name, MAX_RC_NAME_LEN, "pcie%d-long", i);
		msm_pcie_dev[i].ipc_log_long =
			ipc_log_context_create(PCIE_LOG_PAGES, rc_name, 0);
		if (msm_pcie_dev[i].ipc_log_long == NULL)
			pr_err("%s: unable to create IPC log context for %s\n",
				__func__, rc_name);
		else
			PCIE_DBG(&msm_pcie_dev[i],
				"PCIe IPC logging %s is enable for RC%d\n",
				rc_name, i);
		snprintf(rc_name, MAX_RC_NAME_LEN, "pcie%d-dump", i);
		msm_pcie_dev[i].ipc_log_dump =
			ipc_log_context_create(PCIE_LOG_PAGES, rc_name, 0);
		if (msm_pcie_dev[i].ipc_log_dump == NULL)
			pr_err("%s: unable to create IPC log context for %s\n",
				__func__, rc_name);
		else
			PCIE_DBG(&msm_pcie_dev[i],
				"PCIe IPC logging %s is enable for RC%d\n",
				rc_name, i);
		spin_lock_init(&msm_pcie_dev[i].cfg_lock);
		msm_pcie_dev[i].cfg_access = true;
		mutex_init(&msm_pcie_dev[i].enumerate_lock);
		mutex_init(&msm_pcie_dev[i].setup_lock);
		mutex_init(&msm_pcie_dev[i].recovery_lock);
		spin_lock_init(&msm_pcie_dev[i].wakeup_lock);
		spin_lock_init(&msm_pcie_dev[i].irq_lock);
		msm_pcie_dev[i].drv_ready = false;
	}
	for (i = 0; i < MAX_RC_NUM * MAX_DEVICE_NUM; i++) {
		msm_pcie_dev_tbl[i].bdf = 0;
		msm_pcie_dev_tbl[i].dev = NULL;
		msm_pcie_dev_tbl[i].short_bdf = 0;
		msm_pcie_dev_tbl[i].sid = 0;
		msm_pcie_dev_tbl[i].domain = -1;
		msm_pcie_dev_tbl[i].conf_base = NULL;
		msm_pcie_dev_tbl[i].phy_address = 0;
		msm_pcie_dev_tbl[i].dev_ctrlstts_offset = 0;
		msm_pcie_dev_tbl[i].event_reg = NULL;
		msm_pcie_dev_tbl[i].registered = true;
	}

	msm_pcie_debugfs_init();

	ret = platform_driver_register(&msm_pcie_driver);

	return ret;
}

static void __exit pcie_exit(void)
{
	int i;

	PCIE_GEN_DBG("pcie:%s.\n", __func__);

	platform_driver_unregister(&msm_pcie_driver);

	msm_pcie_debugfs_exit();

	for (i = 0; i < MAX_RC_NUM; i++)
		msm_pcie_sysfs_exit(&msm_pcie_dev[i]);
}

subsys_initcall_sync(pcie_init);
module_exit(pcie_exit);


/* RC do not represent the right class; set it to PCI_CLASS_BRIDGE_PCI */
static void msm_pcie_fixup_early(struct pci_dev *dev)
{
	struct msm_pcie_dev_t *pcie_dev = PCIE_BUS_PRIV_DATA(dev->bus);

	PCIE_DBG(pcie_dev, "hdr_type %d\n", dev->hdr_type);
	if (pci_is_root_bus(dev->bus))
		dev->class = (dev->class & 0xff) | (PCI_CLASS_BRIDGE_PCI << 8);
}
DECLARE_PCI_FIXUP_EARLY(PCIE_VENDOR_ID_QCOM, PCI_ANY_ID,
			msm_pcie_fixup_early);

/* Suspend the PCIe link */
static int msm_pcie_pm_suspend(struct pci_dev *dev,
			void *user, void *data, u32 options)
{
	int ret = 0;
	u32 val = 0;
	int ret_l23;
	unsigned long irqsave_flags;
	struct msm_pcie_dev_t *pcie_dev = PCIE_BUS_PRIV_DATA(dev->bus);

	PCIE_DBG(pcie_dev, "RC%d: entry\n", pcie_dev->rc_idx);

	spin_lock_irqsave(&pcie_dev->irq_lock, irqsave_flags);
	pcie_dev->suspending = true;
	spin_unlock_irqrestore(&pcie_dev->irq_lock, irqsave_flags);

	if (!pcie_dev->power_on) {
		PCIE_DBG(pcie_dev,
			"PCIe: power of RC%d has been turned off.\n",
			pcie_dev->rc_idx);
		return ret;
	}

	if (dev && !(options & MSM_PCIE_CONFIG_NO_CFG_RESTORE)
		&& msm_pcie_confirm_linkup(pcie_dev, true, true,
			pcie_dev->conf)) {
		ret = pci_save_state(dev);
		pcie_dev->saved_state =	pci_store_saved_state(dev);
	}
	if (ret) {
		PCIE_ERR(pcie_dev, "PCIe: fail to save state of RC%d:%d.\n",
			pcie_dev->rc_idx, ret);
		pcie_dev->suspending = false;
		return ret;
	}

	spin_lock_irqsave(&pcie_dev->cfg_lock,
				pcie_dev->irqsave_flags);
	pcie_dev->cfg_access = false;
	spin_unlock_irqrestore(&pcie_dev->cfg_lock,
				pcie_dev->irqsave_flags);

	msm_pcie_write_mask(pcie_dev->elbi + PCIE20_ELBI_SYS_CTRL, 0,
				BIT(4));

	PCIE_DBG(pcie_dev, "RC%d: PME_TURNOFF_MSG is sent out\n",
		pcie_dev->rc_idx);

	ret_l23 = readl_poll_timeout((pcie_dev->parf
		+ PCIE20_PARF_PM_STTS), val, (val & BIT(5)), 10000, 100000);

	/* check L23_Ready */
	PCIE_DBG(pcie_dev, "RC%d: PCIE20_PARF_PM_STTS is 0x%x.\n",
		pcie_dev->rc_idx,
		readl_relaxed(pcie_dev->parf + PCIE20_PARF_PM_STTS));
	if (!ret_l23)
		PCIE_DBG(pcie_dev, "RC%d: PM_Enter_L23 is received\n",
			pcie_dev->rc_idx);
	else
		PCIE_DBG(pcie_dev, "RC%d: PM_Enter_L23 is NOT received\n",
			pcie_dev->rc_idx);

	if (pcie_dev->use_pinctrl && pcie_dev->pins_sleep)
		pinctrl_select_state(pcie_dev->pinctrl,
					pcie_dev->pins_sleep);

	msm_pcie_disable(pcie_dev, PM_PIPE_CLK | PM_CLK | PM_VREG);

	PCIE_DBG(pcie_dev, "RC%d: exit\n", pcie_dev->rc_idx);

	return ret;
}

static void msm_pcie_fixup_suspend(struct pci_dev *dev)
{
	int ret;
	struct msm_pcie_dev_t *pcie_dev = PCIE_BUS_PRIV_DATA(dev->bus);

	PCIE_DBG(pcie_dev, "RC%d\n", pcie_dev->rc_idx);

	if (pcie_dev->link_status != MSM_PCIE_LINK_ENABLED ||
		!pci_is_root_bus(dev->bus))
		return;

	spin_lock_irqsave(&pcie_dev->cfg_lock,
				pcie_dev->irqsave_flags);
	if (pcie_dev->disable_pc) {
		PCIE_DBG(pcie_dev,
			"RC%d: Skip suspend because of user request\n",
			pcie_dev->rc_idx);
		spin_unlock_irqrestore(&pcie_dev->cfg_lock,
				pcie_dev->irqsave_flags);
		return;
	}
	spin_unlock_irqrestore(&pcie_dev->cfg_lock,
				pcie_dev->irqsave_flags);

	mutex_lock(&pcie_dev->recovery_lock);

	ret = msm_pcie_pm_suspend(dev, NULL, NULL, 0);
	if (ret)
		PCIE_ERR(pcie_dev, "PCIe: RC%d got failure in suspend:%d.\n",
			pcie_dev->rc_idx, ret);

	mutex_unlock(&pcie_dev->recovery_lock);
}
DECLARE_PCI_FIXUP_SUSPEND(PCIE_VENDOR_ID_QCOM, PCI_ANY_ID,
			  msm_pcie_fixup_suspend);

/* Resume the PCIe link */
static int msm_pcie_pm_resume(struct pci_dev *dev,
			void *user, void *data, u32 options)
{
	int ret;
	struct msm_pcie_dev_t *pcie_dev = PCIE_BUS_PRIV_DATA(dev->bus);

	PCIE_DBG(pcie_dev, "RC%d: entry\n", pcie_dev->rc_idx);

	if (pcie_dev->use_pinctrl && pcie_dev->pins_default)
		pinctrl_select_state(pcie_dev->pinctrl,
					pcie_dev->pins_default);

	spin_lock_irqsave(&pcie_dev->cfg_lock,
				pcie_dev->irqsave_flags);
	pcie_dev->cfg_access = true;
	spin_unlock_irqrestore(&pcie_dev->cfg_lock,
				pcie_dev->irqsave_flags);

	ret = msm_pcie_enable(pcie_dev, PM_PIPE_CLK | PM_CLK | PM_VREG);
	if (ret) {
		PCIE_ERR(pcie_dev,
			"PCIe: RC%d fail to enable PCIe link in resume.\n",
			pcie_dev->rc_idx);
		return ret;
	}

	pcie_dev->suspending = false;
	PCIE_DBG(pcie_dev,
		"dev->bus->number = %d dev->bus->primary = %d\n",
		 dev->bus->number, dev->bus->primary);

	if (!(options & MSM_PCIE_CONFIG_NO_CFG_RESTORE)) {
		PCIE_DBG(pcie_dev,
			"RC%d: entry of PCI framework restore state\n",
			pcie_dev->rc_idx);

		pci_load_and_free_saved_state(dev,
				&pcie_dev->saved_state);
		pci_restore_state(dev);

		PCIE_DBG(pcie_dev,
			"RC%d: exit of PCI framework restore state\n",
			pcie_dev->rc_idx);
	}

	if (pcie_dev->bridge_found) {
		PCIE_DBG(pcie_dev,
			"RC%d: entry of PCIe recover config\n",
			pcie_dev->rc_idx);

		msm_pcie_recover_config(dev);

		PCIE_DBG(pcie_dev,
			"RC%d: exit of PCIe recover config\n",
			pcie_dev->rc_idx);
	}

	PCIE_DBG(pcie_dev, "RC%d: exit\n", pcie_dev->rc_idx);

	return ret;
}

static void msm_pcie_fixup_resume(struct pci_dev *dev)
{
	int ret;
	struct msm_pcie_dev_t *pcie_dev = PCIE_BUS_PRIV_DATA(dev->bus);

	PCIE_DBG(pcie_dev, "RC%d\n", pcie_dev->rc_idx);

	if ((pcie_dev->link_status != MSM_PCIE_LINK_DISABLED) ||
		pcie_dev->user_suspend || !pci_is_root_bus(dev->bus))
		return;

	mutex_lock(&pcie_dev->recovery_lock);
	ret = msm_pcie_pm_resume(dev, NULL, NULL, 0);
	if (ret)
		PCIE_ERR(pcie_dev,
			"PCIe: RC%d got failure in fixup resume:%d.\n",
			pcie_dev->rc_idx, ret);

	mutex_unlock(&pcie_dev->recovery_lock);
}
DECLARE_PCI_FIXUP_RESUME(PCIE_VENDOR_ID_QCOM, PCI_ANY_ID,
				 msm_pcie_fixup_resume);

static void msm_pcie_fixup_resume_early(struct pci_dev *dev)
{
	int ret;
	struct msm_pcie_dev_t *pcie_dev = PCIE_BUS_PRIV_DATA(dev->bus);

	PCIE_DBG(pcie_dev, "RC%d\n", pcie_dev->rc_idx);

	if ((pcie_dev->link_status != MSM_PCIE_LINK_DISABLED) ||
		pcie_dev->user_suspend || !pci_is_root_bus(dev->bus))
		return;

	mutex_lock(&pcie_dev->recovery_lock);
	ret = msm_pcie_pm_resume(dev, NULL, NULL, 0);
	if (ret)
		PCIE_ERR(pcie_dev, "PCIe: RC%d got failure in resume:%d.\n",
			pcie_dev->rc_idx, ret);

	mutex_unlock(&pcie_dev->recovery_lock);
}
DECLARE_PCI_FIXUP_RESUME_EARLY(PCIE_VENDOR_ID_QCOM, PCI_ANY_ID,
				 msm_pcie_fixup_resume_early);

int msm_pcie_pm_control(enum msm_pcie_pm_opt pm_opt, u32 busnr, void *user,
			void *data, u32 options)
{
	int i, ret = 0;
	struct pci_dev *dev;
	u32 rc_idx = 0;
	struct msm_pcie_dev_t *pcie_dev;

	PCIE_GEN_DBG("PCIe: pm_opt:%d;busnr:%d;options:%d\n",
		pm_opt, busnr, options);


	if (!user) {
		pr_err("PCIe: endpoint device is NULL\n");
		ret = -ENODEV;
		goto out;
	}

	pcie_dev = PCIE_BUS_PRIV_DATA(((struct pci_dev *)user)->bus);

	if (pcie_dev) {
		rc_idx = pcie_dev->rc_idx;
		PCIE_DBG(pcie_dev,
			"PCIe: RC%d: pm_opt:%d;busnr:%d;options:%d\n",
			rc_idx, pm_opt, busnr, options);
	} else {
		pr_err(
			"PCIe: did not find RC for pci endpoint device.\n"
			);
		ret = -ENODEV;
		goto out;
	}

	for (i = 0; i < MAX_DEVICE_NUM; i++) {
		if (!busnr)
			break;
		if (user == pcie_dev->pcidev_table[i].dev) {
			if (busnr == pcie_dev->pcidev_table[i].bdf >> 24)
				break;

			PCIE_ERR(pcie_dev,
				"PCIe: RC%d: bus number %d does not match with the expected value %d\n",
				pcie_dev->rc_idx, busnr,
				pcie_dev->pcidev_table[i].bdf >> 24);
			ret = MSM_PCIE_ERROR;
			goto out;
		}
	}

	if (i == MAX_DEVICE_NUM) {
		PCIE_ERR(pcie_dev,
			"PCIe: RC%d: endpoint device was not found in device table",
			pcie_dev->rc_idx);
		ret = MSM_PCIE_ERROR;
		goto out;
	}

	dev = msm_pcie_dev[rc_idx].dev;

	if (!msm_pcie_dev[rc_idx].drv_ready) {
		PCIE_ERR(&msm_pcie_dev[rc_idx],
			"RC%d has not been successfully probed yet\n",
			rc_idx);
		return -EPROBE_DEFER;
	}

	switch (pm_opt) {
	case MSM_PCIE_SUSPEND:
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"User of RC%d requests to suspend the link\n", rc_idx);
		if (msm_pcie_dev[rc_idx].link_status != MSM_PCIE_LINK_ENABLED)
			PCIE_DBG(&msm_pcie_dev[rc_idx],
				"PCIe: RC%d: requested to suspend when link is not enabled:%d.\n",
				rc_idx, msm_pcie_dev[rc_idx].link_status);

		if (!msm_pcie_dev[rc_idx].power_on) {
			PCIE_ERR(&msm_pcie_dev[rc_idx],
				"PCIe: RC%d: requested to suspend when link is powered down:%d.\n",
				rc_idx, msm_pcie_dev[rc_idx].link_status);
			break;
		}

		if (msm_pcie_dev[rc_idx].pending_ep_reg) {
			PCIE_DBG(&msm_pcie_dev[rc_idx],
				"PCIe: RC%d: request to suspend the link is rejected\n",
				rc_idx);
			break;
		}

		if (pcie_dev->num_active_ep) {
			PCIE_DBG(pcie_dev,
				"RC%d: an EP requested to suspend the link, but other EPs are still active: %d\n",
				pcie_dev->rc_idx, pcie_dev->num_active_ep);
			return ret;
		}

		msm_pcie_dev[rc_idx].user_suspend = true;

		mutex_lock(&msm_pcie_dev[rc_idx].recovery_lock);

		ret = msm_pcie_pm_suspend(dev, user, data, options);
		if (ret) {
			PCIE_ERR(&msm_pcie_dev[rc_idx],
				"PCIe: RC%d: user failed to suspend the link.\n",
				rc_idx);
			msm_pcie_dev[rc_idx].user_suspend = false;
		}

		mutex_unlock(&msm_pcie_dev[rc_idx].recovery_lock);
		break;
	case MSM_PCIE_RESUME:
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"User of RC%d requests to resume the link\n", rc_idx);
		if (msm_pcie_dev[rc_idx].link_status !=
					MSM_PCIE_LINK_DISABLED) {
			PCIE_ERR(&msm_pcie_dev[rc_idx],
				"PCIe: RC%d: requested to resume when link is not disabled:%d. Number of active EP(s): %d\n",
				rc_idx, msm_pcie_dev[rc_idx].link_status,
				msm_pcie_dev[rc_idx].num_active_ep);
			break;
		}

		mutex_lock(&msm_pcie_dev[rc_idx].recovery_lock);
		ret = msm_pcie_pm_resume(dev, user, data, options);
		if (ret) {
			PCIE_ERR(&msm_pcie_dev[rc_idx],
				"PCIe: RC%d: user failed to resume the link.\n",
				rc_idx);
		} else {
			PCIE_DBG(&msm_pcie_dev[rc_idx],
				"PCIe: RC%d: user succeeded to resume the link.\n",
				rc_idx);

			msm_pcie_dev[rc_idx].user_suspend = false;
		}

		mutex_unlock(&msm_pcie_dev[rc_idx].recovery_lock);

		break;
	case MSM_PCIE_DISABLE_PC:
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"User of RC%d requests to keep the link always alive.\n",
			rc_idx);
		spin_lock_irqsave(&msm_pcie_dev[rc_idx].cfg_lock,
				msm_pcie_dev[rc_idx].irqsave_flags);
		if (msm_pcie_dev[rc_idx].suspending) {
			PCIE_ERR(&msm_pcie_dev[rc_idx],
				"PCIe: RC%d Link has been suspended before request\n",
				rc_idx);
			ret = MSM_PCIE_ERROR;
		} else {
			msm_pcie_dev[rc_idx].disable_pc = true;
		}
		spin_unlock_irqrestore(&msm_pcie_dev[rc_idx].cfg_lock,
				msm_pcie_dev[rc_idx].irqsave_flags);
		break;
	case MSM_PCIE_ENABLE_PC:
		PCIE_DBG(&msm_pcie_dev[rc_idx],
			"User of RC%d cancels the request of alive link.\n",
			rc_idx);
		spin_lock_irqsave(&msm_pcie_dev[rc_idx].cfg_lock,
				msm_pcie_dev[rc_idx].irqsave_flags);
		msm_pcie_dev[rc_idx].disable_pc = false;
		spin_unlock_irqrestore(&msm_pcie_dev[rc_idx].cfg_lock,
				msm_pcie_dev[rc_idx].irqsave_flags);
		break;
	default:
		PCIE_ERR(&msm_pcie_dev[rc_idx],
			"PCIe: RC%d: unsupported pm operation:%d.\n",
			rc_idx, pm_opt);
		ret = -ENODEV;
		goto out;
	}

out:
	return ret;
}
EXPORT_SYMBOL(msm_pcie_pm_control);

int msm_pcie_register_event(struct msm_pcie_register_event *reg)
{
	int i, ret = 0;
	struct msm_pcie_dev_t *pcie_dev;

	if (!reg) {
		pr_err("PCIe: Event registration is NULL\n");
		return -ENODEV;
	}

	if (!reg->user) {
		pr_err("PCIe: User of event registration is NULL\n");
		return -ENODEV;
	}

	pcie_dev = PCIE_BUS_PRIV_DATA(((struct pci_dev *)reg->user)->bus);

	if (!pcie_dev) {
		PCIE_ERR(pcie_dev, "%s",
			"PCIe: did not find RC for pci endpoint device.\n");
		return -ENODEV;
	}

	if (pcie_dev->num_ep > 1) {
		for (i = 0; i < MAX_DEVICE_NUM; i++) {
			if (reg->user ==
				pcie_dev->pcidev_table[i].dev) {
				pcie_dev->event_reg =
					pcie_dev->pcidev_table[i].event_reg;

				if (!pcie_dev->event_reg) {
					pcie_dev->pcidev_table[i].registered =
						true;

					pcie_dev->num_active_ep++;
					PCIE_DBG(pcie_dev,
						"PCIe: RC%d: number of active EP(s): %d.\n",
						pcie_dev->rc_idx,
						pcie_dev->num_active_ep);
				}

				pcie_dev->event_reg = reg;
				pcie_dev->pcidev_table[i].event_reg = reg;
				PCIE_DBG(pcie_dev,
					"Event 0x%x is registered for RC %d\n",
					reg->events,
					pcie_dev->rc_idx);

				break;
			}
		}

		if (pcie_dev->pending_ep_reg) {
			for (i = 0; i < MAX_DEVICE_NUM; i++)
				if (!pcie_dev->pcidev_table[i].registered)
					break;

			if (i == MAX_DEVICE_NUM)
				pcie_dev->pending_ep_reg = false;
		}
	} else {
		pcie_dev->event_reg = reg;
		PCIE_DBG(pcie_dev,
			"Event 0x%x is registered for RC %d\n", reg->events,
			pcie_dev->rc_idx);
	}

	return ret;
}
EXPORT_SYMBOL(msm_pcie_register_event);

int msm_pcie_deregister_event(struct msm_pcie_register_event *reg)
{
	int i, ret = 0;
	struct msm_pcie_dev_t *pcie_dev;

	if (!reg) {
		pr_err("PCIe: Event deregistration is NULL\n");
		return -ENODEV;
	}

	if (!reg->user) {
		pr_err("PCIe: User of event deregistration is NULL\n");
		return -ENODEV;
	}

	pcie_dev = PCIE_BUS_PRIV_DATA(((struct pci_dev *)reg->user)->bus);

	if (!pcie_dev) {
		PCIE_ERR(pcie_dev, "%s",
			"PCIe: did not find RC for pci endpoint device.\n");
		return -ENODEV;
	}

	if (pcie_dev->num_ep > 1) {
		for (i = 0; i < MAX_DEVICE_NUM; i++) {
			if (reg->user == pcie_dev->pcidev_table[i].dev) {
				if (pcie_dev->pcidev_table[i].event_reg) {
					pcie_dev->num_active_ep--;
					PCIE_DBG(pcie_dev,
						"PCIe: RC%d: number of active EP(s) left: %d.\n",
						pcie_dev->rc_idx,
						pcie_dev->num_active_ep);
				}

				pcie_dev->event_reg = NULL;
				pcie_dev->pcidev_table[i].event_reg = NULL;
				PCIE_DBG(pcie_dev,
					"Event is deregistered for RC %d\n",
					pcie_dev->rc_idx);

				break;
			}
		}
	} else {
		pcie_dev->event_reg = NULL;
		PCIE_DBG(pcie_dev, "Event is deregistered for RC %d\n",
				pcie_dev->rc_idx);
	}

	return ret;
}
EXPORT_SYMBOL(msm_pcie_deregister_event);

int msm_pcie_recover_config(struct pci_dev *dev)
{
	int ret = 0;
	struct msm_pcie_dev_t *pcie_dev;

	if (dev) {
		pcie_dev = PCIE_BUS_PRIV_DATA(dev->bus);
		PCIE_DBG(pcie_dev,
			"Recovery for the link of RC%d\n", pcie_dev->rc_idx);
	} else {
		pr_err("PCIe: the input pci dev is NULL.\n");
		return -ENODEV;
	}

	if (msm_pcie_confirm_linkup(pcie_dev, true, true, pcie_dev->conf)) {
		PCIE_DBG(pcie_dev,
			"Recover config space of RC%d and its EP\n",
			pcie_dev->rc_idx);
		pcie_dev->shadow_en = false;
		PCIE_DBG(pcie_dev, "Recover RC%d\n", pcie_dev->rc_idx);
		msm_pcie_cfg_recover(pcie_dev, true);
		PCIE_DBG(pcie_dev, "Recover EP of RC%d\n", pcie_dev->rc_idx);
		msm_pcie_cfg_recover(pcie_dev, false);
		PCIE_DBG(pcie_dev,
			"Refreshing the saved config space in PCI framework for RC%d and its EP\n",
			pcie_dev->rc_idx);
		pci_save_state(pcie_dev->dev);
		pci_save_state(dev);
		pcie_dev->shadow_en = true;
		PCIE_DBG(pcie_dev, "Turn on shadow for RC%d\n",
			pcie_dev->rc_idx);
	} else {
		PCIE_ERR(pcie_dev,
			"PCIe: the link of RC%d is not up yet; can't recover config space.\n",
			pcie_dev->rc_idx);
		ret = -ENODEV;
	}

	return ret;
}
EXPORT_SYMBOL(msm_pcie_recover_config);

int msm_pcie_shadow_control(struct pci_dev *dev, bool enable)
{
	int ret = 0;
	struct msm_pcie_dev_t *pcie_dev;

	if (dev) {
		pcie_dev = PCIE_BUS_PRIV_DATA(dev->bus);
		PCIE_DBG(pcie_dev,
			"User requests to %s shadow\n",
			enable ? "enable" : "disable");
	} else {
		pr_err("PCIe: the input pci dev is NULL.\n");
		return -ENODEV;
	}

	PCIE_DBG(pcie_dev,
		"The shadowing of RC%d is %s enabled currently.\n",
		pcie_dev->rc_idx, pcie_dev->shadow_en ? "" : "not");

	pcie_dev->shadow_en = enable;

	PCIE_DBG(pcie_dev,
		"Shadowing of RC%d is turned %s upon user's request.\n",
		pcie_dev->rc_idx, enable ? "on" : "off");

	return ret;
}
EXPORT_SYMBOL(msm_pcie_shadow_control);