blob: b33be3ef25ae2284840217fa86fcfd540f607eec [file] [log] [blame]
Marcin Wojtas3f518502014-07-10 16:52:13 -03001/*
2 * Driver for Marvell PPv2 network controller for Armada 375 SoC.
3 *
4 * Copyright (C) 2014 Marvell
5 *
6 * Marcin Wojtas <mw@semihalf.com>
7 *
8 * This file is licensed under the terms of the GNU General Public
9 * License version 2. This program is licensed "as is" without any
10 * warranty of any kind, whether express or implied.
11 */
12
13#include <linux/kernel.h>
14#include <linux/netdevice.h>
15#include <linux/etherdevice.h>
16#include <linux/platform_device.h>
17#include <linux/skbuff.h>
18#include <linux/inetdevice.h>
19#include <linux/mbus.h>
20#include <linux/module.h>
Antoine Ténartf84bf382017-08-22 19:08:27 +020021#include <linux/mfd/syscon.h>
Marcin Wojtas3f518502014-07-10 16:52:13 -030022#include <linux/interrupt.h>
23#include <linux/cpumask.h>
24#include <linux/of.h>
25#include <linux/of_irq.h>
26#include <linux/of_mdio.h>
27#include <linux/of_net.h>
28#include <linux/of_address.h>
Thomas Petazzonifaca9242017-03-07 16:53:06 +010029#include <linux/of_device.h>
Marcin Wojtas3f518502014-07-10 16:52:13 -030030#include <linux/phy.h>
Antoine Tenart542897d2017-08-30 10:29:15 +020031#include <linux/phy/phy.h>
Marcin Wojtas3f518502014-07-10 16:52:13 -030032#include <linux/clk.h>
Marcin Wojtasedc660f2015-08-06 19:00:30 +020033#include <linux/hrtimer.h>
34#include <linux/ktime.h>
Antoine Ténartf84bf382017-08-22 19:08:27 +020035#include <linux/regmap.h>
Marcin Wojtas3f518502014-07-10 16:52:13 -030036#include <uapi/linux/ppp_defs.h>
37#include <net/ip.h>
38#include <net/ipv6.h>
Antoine Ténart186cd4d2017-08-23 09:46:56 +020039#include <net/tso.h>
Marcin Wojtas3f518502014-07-10 16:52:13 -030040
41/* RX Fifo Registers */
42#define MVPP2_RX_DATA_FIFO_SIZE_REG(port) (0x00 + 4 * (port))
43#define MVPP2_RX_ATTR_FIFO_SIZE_REG(port) (0x20 + 4 * (port))
44#define MVPP2_RX_MIN_PKT_SIZE_REG 0x60
45#define MVPP2_RX_FIFO_INIT_REG 0x64
46
47/* RX DMA Top Registers */
48#define MVPP2_RX_CTRL_REG(port) (0x140 + 4 * (port))
49#define MVPP2_RX_LOW_LATENCY_PKT_SIZE(s) (((s) & 0xfff) << 16)
50#define MVPP2_RX_USE_PSEUDO_FOR_CSUM_MASK BIT(31)
51#define MVPP2_POOL_BUF_SIZE_REG(pool) (0x180 + 4 * (pool))
52#define MVPP2_POOL_BUF_SIZE_OFFSET 5
53#define MVPP2_RXQ_CONFIG_REG(rxq) (0x800 + 4 * (rxq))
54#define MVPP2_SNOOP_PKT_SIZE_MASK 0x1ff
55#define MVPP2_SNOOP_BUF_HDR_MASK BIT(9)
56#define MVPP2_RXQ_POOL_SHORT_OFFS 20
Thomas Petazzoni5eac8922017-03-07 16:53:10 +010057#define MVPP21_RXQ_POOL_SHORT_MASK 0x700000
58#define MVPP22_RXQ_POOL_SHORT_MASK 0xf00000
Marcin Wojtas3f518502014-07-10 16:52:13 -030059#define MVPP2_RXQ_POOL_LONG_OFFS 24
Thomas Petazzoni5eac8922017-03-07 16:53:10 +010060#define MVPP21_RXQ_POOL_LONG_MASK 0x7000000
61#define MVPP22_RXQ_POOL_LONG_MASK 0xf000000
Marcin Wojtas3f518502014-07-10 16:52:13 -030062#define MVPP2_RXQ_PACKET_OFFSET_OFFS 28
63#define MVPP2_RXQ_PACKET_OFFSET_MASK 0x70000000
64#define MVPP2_RXQ_DISABLE_MASK BIT(31)
65
66/* Parser Registers */
67#define MVPP2_PRS_INIT_LOOKUP_REG 0x1000
68#define MVPP2_PRS_PORT_LU_MAX 0xf
69#define MVPP2_PRS_PORT_LU_MASK(port) (0xff << ((port) * 4))
70#define MVPP2_PRS_PORT_LU_VAL(port, val) ((val) << ((port) * 4))
71#define MVPP2_PRS_INIT_OFFS_REG(port) (0x1004 + ((port) & 4))
72#define MVPP2_PRS_INIT_OFF_MASK(port) (0x3f << (((port) % 4) * 8))
73#define MVPP2_PRS_INIT_OFF_VAL(port, val) ((val) << (((port) % 4) * 8))
74#define MVPP2_PRS_MAX_LOOP_REG(port) (0x100c + ((port) & 4))
75#define MVPP2_PRS_MAX_LOOP_MASK(port) (0xff << (((port) % 4) * 8))
76#define MVPP2_PRS_MAX_LOOP_VAL(port, val) ((val) << (((port) % 4) * 8))
77#define MVPP2_PRS_TCAM_IDX_REG 0x1100
78#define MVPP2_PRS_TCAM_DATA_REG(idx) (0x1104 + (idx) * 4)
79#define MVPP2_PRS_TCAM_INV_MASK BIT(31)
80#define MVPP2_PRS_SRAM_IDX_REG 0x1200
81#define MVPP2_PRS_SRAM_DATA_REG(idx) (0x1204 + (idx) * 4)
82#define MVPP2_PRS_TCAM_CTRL_REG 0x1230
83#define MVPP2_PRS_TCAM_EN_MASK BIT(0)
84
85/* Classifier Registers */
86#define MVPP2_CLS_MODE_REG 0x1800
87#define MVPP2_CLS_MODE_ACTIVE_MASK BIT(0)
88#define MVPP2_CLS_PORT_WAY_REG 0x1810
89#define MVPP2_CLS_PORT_WAY_MASK(port) (1 << (port))
90#define MVPP2_CLS_LKP_INDEX_REG 0x1814
91#define MVPP2_CLS_LKP_INDEX_WAY_OFFS 6
92#define MVPP2_CLS_LKP_TBL_REG 0x1818
93#define MVPP2_CLS_LKP_TBL_RXQ_MASK 0xff
94#define MVPP2_CLS_LKP_TBL_LOOKUP_EN_MASK BIT(25)
95#define MVPP2_CLS_FLOW_INDEX_REG 0x1820
96#define MVPP2_CLS_FLOW_TBL0_REG 0x1824
97#define MVPP2_CLS_FLOW_TBL1_REG 0x1828
98#define MVPP2_CLS_FLOW_TBL2_REG 0x182c
99#define MVPP2_CLS_OVERSIZE_RXQ_LOW_REG(port) (0x1980 + ((port) * 4))
100#define MVPP2_CLS_OVERSIZE_RXQ_LOW_BITS 3
101#define MVPP2_CLS_OVERSIZE_RXQ_LOW_MASK 0x7
102#define MVPP2_CLS_SWFWD_P2HQ_REG(port) (0x19b0 + ((port) * 4))
103#define MVPP2_CLS_SWFWD_PCTRL_REG 0x19d0
104#define MVPP2_CLS_SWFWD_PCTRL_MASK(port) (1 << (port))
105
106/* Descriptor Manager Top Registers */
107#define MVPP2_RXQ_NUM_REG 0x2040
108#define MVPP2_RXQ_DESC_ADDR_REG 0x2044
Thomas Petazzonib02f31f2017-03-07 16:53:12 +0100109#define MVPP22_DESC_ADDR_OFFS 8
Marcin Wojtas3f518502014-07-10 16:52:13 -0300110#define MVPP2_RXQ_DESC_SIZE_REG 0x2048
111#define MVPP2_RXQ_DESC_SIZE_MASK 0x3ff0
112#define MVPP2_RXQ_STATUS_UPDATE_REG(rxq) (0x3000 + 4 * (rxq))
113#define MVPP2_RXQ_NUM_PROCESSED_OFFSET 0
114#define MVPP2_RXQ_NUM_NEW_OFFSET 16
115#define MVPP2_RXQ_STATUS_REG(rxq) (0x3400 + 4 * (rxq))
116#define MVPP2_RXQ_OCCUPIED_MASK 0x3fff
117#define MVPP2_RXQ_NON_OCCUPIED_OFFSET 16
118#define MVPP2_RXQ_NON_OCCUPIED_MASK 0x3fff0000
119#define MVPP2_RXQ_THRESH_REG 0x204c
120#define MVPP2_OCCUPIED_THRESH_OFFSET 0
121#define MVPP2_OCCUPIED_THRESH_MASK 0x3fff
122#define MVPP2_RXQ_INDEX_REG 0x2050
123#define MVPP2_TXQ_NUM_REG 0x2080
124#define MVPP2_TXQ_DESC_ADDR_REG 0x2084
125#define MVPP2_TXQ_DESC_SIZE_REG 0x2088
126#define MVPP2_TXQ_DESC_SIZE_MASK 0x3ff0
Thomas Petazzoni213f4282017-08-03 10:42:00 +0200127#define MVPP2_TXQ_THRESH_REG 0x2094
128#define MVPP2_TXQ_THRESH_OFFSET 16
129#define MVPP2_TXQ_THRESH_MASK 0x3fff
Marcin Wojtas3f518502014-07-10 16:52:13 -0300130#define MVPP2_AGGR_TXQ_UPDATE_REG 0x2090
Marcin Wojtas3f518502014-07-10 16:52:13 -0300131#define MVPP2_TXQ_INDEX_REG 0x2098
132#define MVPP2_TXQ_PREF_BUF_REG 0x209c
133#define MVPP2_PREF_BUF_PTR(desc) ((desc) & 0xfff)
134#define MVPP2_PREF_BUF_SIZE_4 (BIT(12) | BIT(13))
135#define MVPP2_PREF_BUF_SIZE_16 (BIT(12) | BIT(14))
136#define MVPP2_PREF_BUF_THRESH(val) ((val) << 17)
137#define MVPP2_TXQ_DRAIN_EN_MASK BIT(31)
138#define MVPP2_TXQ_PENDING_REG 0x20a0
139#define MVPP2_TXQ_PENDING_MASK 0x3fff
140#define MVPP2_TXQ_INT_STATUS_REG 0x20a4
141#define MVPP2_TXQ_SENT_REG(txq) (0x3c00 + 4 * (txq))
142#define MVPP2_TRANSMITTED_COUNT_OFFSET 16
143#define MVPP2_TRANSMITTED_COUNT_MASK 0x3fff0000
144#define MVPP2_TXQ_RSVD_REQ_REG 0x20b0
145#define MVPP2_TXQ_RSVD_REQ_Q_OFFSET 16
146#define MVPP2_TXQ_RSVD_RSLT_REG 0x20b4
147#define MVPP2_TXQ_RSVD_RSLT_MASK 0x3fff
148#define MVPP2_TXQ_RSVD_CLR_REG 0x20b8
149#define MVPP2_TXQ_RSVD_CLR_OFFSET 16
150#define MVPP2_AGGR_TXQ_DESC_ADDR_REG(cpu) (0x2100 + 4 * (cpu))
Thomas Petazzonib02f31f2017-03-07 16:53:12 +0100151#define MVPP22_AGGR_TXQ_DESC_ADDR_OFFS 8
Marcin Wojtas3f518502014-07-10 16:52:13 -0300152#define MVPP2_AGGR_TXQ_DESC_SIZE_REG(cpu) (0x2140 + 4 * (cpu))
153#define MVPP2_AGGR_TXQ_DESC_SIZE_MASK 0x3ff0
154#define MVPP2_AGGR_TXQ_STATUS_REG(cpu) (0x2180 + 4 * (cpu))
155#define MVPP2_AGGR_TXQ_PENDING_MASK 0x3fff
156#define MVPP2_AGGR_TXQ_INDEX_REG(cpu) (0x21c0 + 4 * (cpu))
157
158/* MBUS bridge registers */
159#define MVPP2_WIN_BASE(w) (0x4000 + ((w) << 2))
160#define MVPP2_WIN_SIZE(w) (0x4020 + ((w) << 2))
161#define MVPP2_WIN_REMAP(w) (0x4040 + ((w) << 2))
162#define MVPP2_BASE_ADDR_ENABLE 0x4060
163
Thomas Petazzoni6763ce32017-03-07 16:53:15 +0100164/* AXI Bridge Registers */
165#define MVPP22_AXI_BM_WR_ATTR_REG 0x4100
166#define MVPP22_AXI_BM_RD_ATTR_REG 0x4104
167#define MVPP22_AXI_AGGRQ_DESCR_RD_ATTR_REG 0x4110
168#define MVPP22_AXI_TXQ_DESCR_WR_ATTR_REG 0x4114
169#define MVPP22_AXI_TXQ_DESCR_RD_ATTR_REG 0x4118
170#define MVPP22_AXI_RXQ_DESCR_WR_ATTR_REG 0x411c
171#define MVPP22_AXI_RX_DATA_WR_ATTR_REG 0x4120
172#define MVPP22_AXI_TX_DATA_RD_ATTR_REG 0x4130
173#define MVPP22_AXI_RD_NORMAL_CODE_REG 0x4150
174#define MVPP22_AXI_RD_SNOOP_CODE_REG 0x4154
175#define MVPP22_AXI_WR_NORMAL_CODE_REG 0x4160
176#define MVPP22_AXI_WR_SNOOP_CODE_REG 0x4164
177
178/* Values for AXI Bridge registers */
179#define MVPP22_AXI_ATTR_CACHE_OFFS 0
180#define MVPP22_AXI_ATTR_DOMAIN_OFFS 12
181
182#define MVPP22_AXI_CODE_CACHE_OFFS 0
183#define MVPP22_AXI_CODE_DOMAIN_OFFS 4
184
185#define MVPP22_AXI_CODE_CACHE_NON_CACHE 0x3
186#define MVPP22_AXI_CODE_CACHE_WR_CACHE 0x7
187#define MVPP22_AXI_CODE_CACHE_RD_CACHE 0xb
188
189#define MVPP22_AXI_CODE_DOMAIN_OUTER_DOM 2
190#define MVPP22_AXI_CODE_DOMAIN_SYSTEM 3
191
Marcin Wojtas3f518502014-07-10 16:52:13 -0300192/* Interrupt Cause and Mask registers */
Thomas Petazzoni213f4282017-08-03 10:42:00 +0200193#define MVPP2_ISR_TX_THRESHOLD_REG(port) (0x5140 + 4 * (port))
194#define MVPP2_MAX_ISR_TX_THRESHOLD 0xfffff0
195
Marcin Wojtas3f518502014-07-10 16:52:13 -0300196#define MVPP2_ISR_RX_THRESHOLD_REG(rxq) (0x5200 + 4 * (rxq))
Thomas Petazzoniab426762017-02-21 11:28:04 +0100197#define MVPP2_MAX_ISR_RX_THRESHOLD 0xfffff0
Thomas Petazzonieb1e93a2017-08-03 10:41:55 +0200198#define MVPP21_ISR_RXQ_GROUP_REG(port) (0x5400 + 4 * (port))
Thomas Petazzonia73fef12017-03-07 16:53:16 +0100199
Antoine Ténart81b66302017-08-22 19:08:21 +0200200#define MVPP22_ISR_RXQ_GROUP_INDEX_REG 0x5400
Thomas Petazzonia73fef12017-03-07 16:53:16 +0100201#define MVPP22_ISR_RXQ_GROUP_INDEX_SUBGROUP_MASK 0xf
Antoine Ténart81b66302017-08-22 19:08:21 +0200202#define MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_MASK 0x380
203#define MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_OFFSET 7
Thomas Petazzonia73fef12017-03-07 16:53:16 +0100204
205#define MVPP22_ISR_RXQ_GROUP_INDEX_SUBGROUP_MASK 0xf
Antoine Ténart81b66302017-08-22 19:08:21 +0200206#define MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_MASK 0x380
Thomas Petazzonia73fef12017-03-07 16:53:16 +0100207
Antoine Ténart81b66302017-08-22 19:08:21 +0200208#define MVPP22_ISR_RXQ_SUB_GROUP_CONFIG_REG 0x5404
209#define MVPP22_ISR_RXQ_SUB_GROUP_STARTQ_MASK 0x1f
210#define MVPP22_ISR_RXQ_SUB_GROUP_SIZE_MASK 0xf00
211#define MVPP22_ISR_RXQ_SUB_GROUP_SIZE_OFFSET 8
Thomas Petazzonia73fef12017-03-07 16:53:16 +0100212
Marcin Wojtas3f518502014-07-10 16:52:13 -0300213#define MVPP2_ISR_ENABLE_REG(port) (0x5420 + 4 * (port))
214#define MVPP2_ISR_ENABLE_INTERRUPT(mask) ((mask) & 0xffff)
215#define MVPP2_ISR_DISABLE_INTERRUPT(mask) (((mask) << 16) & 0xffff0000)
216#define MVPP2_ISR_RX_TX_CAUSE_REG(port) (0x5480 + 4 * (port))
217#define MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK 0xffff
218#define MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_MASK 0xff0000
Thomas Petazzoni213f4282017-08-03 10:42:00 +0200219#define MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_OFFSET 16
Marcin Wojtas3f518502014-07-10 16:52:13 -0300220#define MVPP2_CAUSE_RX_FIFO_OVERRUN_MASK BIT(24)
221#define MVPP2_CAUSE_FCS_ERR_MASK BIT(25)
222#define MVPP2_CAUSE_TX_FIFO_UNDERRUN_MASK BIT(26)
223#define MVPP2_CAUSE_TX_EXCEPTION_SUM_MASK BIT(29)
224#define MVPP2_CAUSE_RX_EXCEPTION_SUM_MASK BIT(30)
225#define MVPP2_CAUSE_MISC_SUM_MASK BIT(31)
226#define MVPP2_ISR_RX_TX_MASK_REG(port) (0x54a0 + 4 * (port))
227#define MVPP2_ISR_PON_RX_TX_MASK_REG 0x54bc
228#define MVPP2_PON_CAUSE_RXQ_OCCUP_DESC_ALL_MASK 0xffff
229#define MVPP2_PON_CAUSE_TXP_OCCUP_DESC_ALL_MASK 0x3fc00000
230#define MVPP2_PON_CAUSE_MISC_SUM_MASK BIT(31)
231#define MVPP2_ISR_MISC_CAUSE_REG 0x55b0
232
233/* Buffer Manager registers */
234#define MVPP2_BM_POOL_BASE_REG(pool) (0x6000 + ((pool) * 4))
235#define MVPP2_BM_POOL_BASE_ADDR_MASK 0xfffff80
236#define MVPP2_BM_POOL_SIZE_REG(pool) (0x6040 + ((pool) * 4))
237#define MVPP2_BM_POOL_SIZE_MASK 0xfff0
238#define MVPP2_BM_POOL_READ_PTR_REG(pool) (0x6080 + ((pool) * 4))
239#define MVPP2_BM_POOL_GET_READ_PTR_MASK 0xfff0
240#define MVPP2_BM_POOL_PTRS_NUM_REG(pool) (0x60c0 + ((pool) * 4))
241#define MVPP2_BM_POOL_PTRS_NUM_MASK 0xfff0
242#define MVPP2_BM_BPPI_READ_PTR_REG(pool) (0x6100 + ((pool) * 4))
243#define MVPP2_BM_BPPI_PTRS_NUM_REG(pool) (0x6140 + ((pool) * 4))
244#define MVPP2_BM_BPPI_PTR_NUM_MASK 0x7ff
245#define MVPP2_BM_BPPI_PREFETCH_FULL_MASK BIT(16)
246#define MVPP2_BM_POOL_CTRL_REG(pool) (0x6200 + ((pool) * 4))
247#define MVPP2_BM_START_MASK BIT(0)
248#define MVPP2_BM_STOP_MASK BIT(1)
249#define MVPP2_BM_STATE_MASK BIT(4)
250#define MVPP2_BM_LOW_THRESH_OFFS 8
251#define MVPP2_BM_LOW_THRESH_MASK 0x7f00
252#define MVPP2_BM_LOW_THRESH_VALUE(val) ((val) << \
253 MVPP2_BM_LOW_THRESH_OFFS)
254#define MVPP2_BM_HIGH_THRESH_OFFS 16
255#define MVPP2_BM_HIGH_THRESH_MASK 0x7f0000
256#define MVPP2_BM_HIGH_THRESH_VALUE(val) ((val) << \
257 MVPP2_BM_HIGH_THRESH_OFFS)
258#define MVPP2_BM_INTR_CAUSE_REG(pool) (0x6240 + ((pool) * 4))
259#define MVPP2_BM_RELEASED_DELAY_MASK BIT(0)
260#define MVPP2_BM_ALLOC_FAILED_MASK BIT(1)
261#define MVPP2_BM_BPPE_EMPTY_MASK BIT(2)
262#define MVPP2_BM_BPPE_FULL_MASK BIT(3)
263#define MVPP2_BM_AVAILABLE_BP_LOW_MASK BIT(4)
264#define MVPP2_BM_INTR_MASK_REG(pool) (0x6280 + ((pool) * 4))
265#define MVPP2_BM_PHY_ALLOC_REG(pool) (0x6400 + ((pool) * 4))
266#define MVPP2_BM_PHY_ALLOC_GRNTD_MASK BIT(0)
267#define MVPP2_BM_VIRT_ALLOC_REG 0x6440
Thomas Petazzonid01524d2017-03-07 16:53:09 +0100268#define MVPP22_BM_ADDR_HIGH_ALLOC 0x6444
269#define MVPP22_BM_ADDR_HIGH_PHYS_MASK 0xff
270#define MVPP22_BM_ADDR_HIGH_VIRT_MASK 0xff00
271#define MVPP22_BM_ADDR_HIGH_VIRT_SHIFT 8
Marcin Wojtas3f518502014-07-10 16:52:13 -0300272#define MVPP2_BM_PHY_RLS_REG(pool) (0x6480 + ((pool) * 4))
273#define MVPP2_BM_PHY_RLS_MC_BUFF_MASK BIT(0)
274#define MVPP2_BM_PHY_RLS_PRIO_EN_MASK BIT(1)
275#define MVPP2_BM_PHY_RLS_GRNTD_MASK BIT(2)
276#define MVPP2_BM_VIRT_RLS_REG 0x64c0
Thomas Petazzonid01524d2017-03-07 16:53:09 +0100277#define MVPP22_BM_ADDR_HIGH_RLS_REG 0x64c4
278#define MVPP22_BM_ADDR_HIGH_PHYS_RLS_MASK 0xff
Antoine Ténart81b66302017-08-22 19:08:21 +0200279#define MVPP22_BM_ADDR_HIGH_VIRT_RLS_MASK 0xff00
Thomas Petazzonid01524d2017-03-07 16:53:09 +0100280#define MVPP22_BM_ADDR_HIGH_VIRT_RLS_SHIFT 8
Marcin Wojtas3f518502014-07-10 16:52:13 -0300281
282/* TX Scheduler registers */
283#define MVPP2_TXP_SCHED_PORT_INDEX_REG 0x8000
284#define MVPP2_TXP_SCHED_Q_CMD_REG 0x8004
285#define MVPP2_TXP_SCHED_ENQ_MASK 0xff
286#define MVPP2_TXP_SCHED_DISQ_OFFSET 8
287#define MVPP2_TXP_SCHED_CMD_1_REG 0x8010
288#define MVPP2_TXP_SCHED_PERIOD_REG 0x8018
289#define MVPP2_TXP_SCHED_MTU_REG 0x801c
290#define MVPP2_TXP_MTU_MAX 0x7FFFF
291#define MVPP2_TXP_SCHED_REFILL_REG 0x8020
292#define MVPP2_TXP_REFILL_TOKENS_ALL_MASK 0x7ffff
293#define MVPP2_TXP_REFILL_PERIOD_ALL_MASK 0x3ff00000
294#define MVPP2_TXP_REFILL_PERIOD_MASK(v) ((v) << 20)
295#define MVPP2_TXP_SCHED_TOKEN_SIZE_REG 0x8024
296#define MVPP2_TXP_TOKEN_SIZE_MAX 0xffffffff
297#define MVPP2_TXQ_SCHED_REFILL_REG(q) (0x8040 + ((q) << 2))
298#define MVPP2_TXQ_REFILL_TOKENS_ALL_MASK 0x7ffff
299#define MVPP2_TXQ_REFILL_PERIOD_ALL_MASK 0x3ff00000
300#define MVPP2_TXQ_REFILL_PERIOD_MASK(v) ((v) << 20)
301#define MVPP2_TXQ_SCHED_TOKEN_SIZE_REG(q) (0x8060 + ((q) << 2))
302#define MVPP2_TXQ_TOKEN_SIZE_MAX 0x7fffffff
303#define MVPP2_TXQ_SCHED_TOKEN_CNTR_REG(q) (0x8080 + ((q) << 2))
304#define MVPP2_TXQ_TOKEN_CNTR_MAX 0xffffffff
305
306/* TX general registers */
307#define MVPP2_TX_SNOOP_REG 0x8800
308#define MVPP2_TX_PORT_FLUSH_REG 0x8810
309#define MVPP2_TX_PORT_FLUSH_MASK(port) (1 << (port))
310
311/* LMS registers */
312#define MVPP2_SRC_ADDR_MIDDLE 0x24
313#define MVPP2_SRC_ADDR_HIGH 0x28
Marcin Wojtas08a23752014-07-21 13:48:12 -0300314#define MVPP2_PHY_AN_CFG0_REG 0x34
315#define MVPP2_PHY_AN_STOP_SMI0_MASK BIT(7)
Marcin Wojtas3f518502014-07-10 16:52:13 -0300316#define MVPP2_MNG_EXTENDED_GLOBAL_CTRL_REG 0x305c
Thomas Petazzoni31d76772017-02-21 11:28:10 +0100317#define MVPP2_EXT_GLOBAL_CTRL_DEFAULT 0x27
Marcin Wojtas3f518502014-07-10 16:52:13 -0300318
319/* Per-port registers */
320#define MVPP2_GMAC_CTRL_0_REG 0x0
Antoine Ténart81b66302017-08-22 19:08:21 +0200321#define MVPP2_GMAC_PORT_EN_MASK BIT(0)
Antoine Ténart39193572017-08-22 19:08:24 +0200322#define MVPP2_GMAC_PORT_TYPE_MASK BIT(1)
Antoine Ténart81b66302017-08-22 19:08:21 +0200323#define MVPP2_GMAC_MAX_RX_SIZE_OFFS 2
324#define MVPP2_GMAC_MAX_RX_SIZE_MASK 0x7ffc
325#define MVPP2_GMAC_MIB_CNTR_EN_MASK BIT(15)
Marcin Wojtas3f518502014-07-10 16:52:13 -0300326#define MVPP2_GMAC_CTRL_1_REG 0x4
Antoine Ténart81b66302017-08-22 19:08:21 +0200327#define MVPP2_GMAC_PERIODIC_XON_EN_MASK BIT(1)
328#define MVPP2_GMAC_GMII_LB_EN_MASK BIT(5)
329#define MVPP2_GMAC_PCS_LB_EN_BIT 6
330#define MVPP2_GMAC_PCS_LB_EN_MASK BIT(6)
331#define MVPP2_GMAC_SA_LOW_OFFS 7
Marcin Wojtas3f518502014-07-10 16:52:13 -0300332#define MVPP2_GMAC_CTRL_2_REG 0x8
Antoine Ténart81b66302017-08-22 19:08:21 +0200333#define MVPP2_GMAC_INBAND_AN_MASK BIT(0)
Antoine Ténart39193572017-08-22 19:08:24 +0200334#define MVPP2_GMAC_FLOW_CTRL_MASK GENMASK(2, 1)
Antoine Ténart81b66302017-08-22 19:08:21 +0200335#define MVPP2_GMAC_PCS_ENABLE_MASK BIT(3)
336#define MVPP2_GMAC_PORT_RGMII_MASK BIT(4)
Antoine Ténart39193572017-08-22 19:08:24 +0200337#define MVPP2_GMAC_DISABLE_PADDING BIT(5)
Antoine Ténart81b66302017-08-22 19:08:21 +0200338#define MVPP2_GMAC_PORT_RESET_MASK BIT(6)
Marcin Wojtas3f518502014-07-10 16:52:13 -0300339#define MVPP2_GMAC_AUTONEG_CONFIG 0xc
Antoine Ténart81b66302017-08-22 19:08:21 +0200340#define MVPP2_GMAC_FORCE_LINK_DOWN BIT(0)
341#define MVPP2_GMAC_FORCE_LINK_PASS BIT(1)
Antoine Ténart39193572017-08-22 19:08:24 +0200342#define MVPP2_GMAC_IN_BAND_AUTONEG BIT(2)
343#define MVPP2_GMAC_IN_BAND_AUTONEG_BYPASS BIT(3)
Antoine Ténart81b66302017-08-22 19:08:21 +0200344#define MVPP2_GMAC_CONFIG_MII_SPEED BIT(5)
345#define MVPP2_GMAC_CONFIG_GMII_SPEED BIT(6)
346#define MVPP2_GMAC_AN_SPEED_EN BIT(7)
347#define MVPP2_GMAC_FC_ADV_EN BIT(9)
Antoine Ténart39193572017-08-22 19:08:24 +0200348#define MVPP2_GMAC_FLOW_CTRL_AUTONEG BIT(11)
Antoine Ténart81b66302017-08-22 19:08:21 +0200349#define MVPP2_GMAC_CONFIG_FULL_DUPLEX BIT(12)
350#define MVPP2_GMAC_AN_DUPLEX_EN BIT(13)
Marcin Wojtas3f518502014-07-10 16:52:13 -0300351#define MVPP2_GMAC_PORT_FIFO_CFG_1_REG 0x1c
Antoine Ténart81b66302017-08-22 19:08:21 +0200352#define MVPP2_GMAC_TX_FIFO_MIN_TH_OFFS 6
353#define MVPP2_GMAC_TX_FIFO_MIN_TH_ALL_MASK 0x1fc0
354#define MVPP2_GMAC_TX_FIFO_MIN_TH_MASK(v) (((v) << 6) & \
Marcin Wojtas3f518502014-07-10 16:52:13 -0300355 MVPP2_GMAC_TX_FIFO_MIN_TH_ALL_MASK)
Thomas Petazzoni26975822017-03-07 16:53:14 +0100356#define MVPP22_GMAC_CTRL_4_REG 0x90
Antoine Ténart81b66302017-08-22 19:08:21 +0200357#define MVPP22_CTRL4_EXT_PIN_GMII_SEL BIT(0)
358#define MVPP22_CTRL4_DP_CLK_SEL BIT(5)
Antoine Ténart1068ec72017-08-22 19:08:22 +0200359#define MVPP22_CTRL4_SYNC_BYPASS_DIS BIT(6)
Antoine Ténart81b66302017-08-22 19:08:21 +0200360#define MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE BIT(7)
Thomas Petazzoni26975822017-03-07 16:53:14 +0100361
362/* Per-port XGMAC registers. PPv2.2 only, only for GOP port 0,
363 * relative to port->base.
364 */
Antoine Ténart725757a2017-06-12 16:01:39 +0200365#define MVPP22_XLG_CTRL0_REG 0x100
Antoine Ténart81b66302017-08-22 19:08:21 +0200366#define MVPP22_XLG_CTRL0_PORT_EN BIT(0)
367#define MVPP22_XLG_CTRL0_MAC_RESET_DIS BIT(1)
Antoine Ténart77321952017-08-22 19:08:25 +0200368#define MVPP22_XLG_CTRL0_RX_FLOW_CTRL_EN BIT(7)
Antoine Ténart81b66302017-08-22 19:08:21 +0200369#define MVPP22_XLG_CTRL0_MIB_CNT_DIS BIT(14)
Stefan Chulski76eb1b12017-08-22 19:08:26 +0200370#define MVPP22_XLG_CTRL1_REG 0x104
Antoine Ténartec15ecd2017-08-25 15:24:46 +0200371#define MVPP22_XLG_CTRL1_FRAMESIZELIMIT_OFFS 0
Stefan Chulski76eb1b12017-08-22 19:08:26 +0200372#define MVPP22_XLG_CTRL1_FRAMESIZELIMIT_MASK 0x1fff
Thomas Petazzoni26975822017-03-07 16:53:14 +0100373#define MVPP22_XLG_CTRL3_REG 0x11c
Antoine Ténart81b66302017-08-22 19:08:21 +0200374#define MVPP22_XLG_CTRL3_MACMODESELECT_MASK (7 << 13)
375#define MVPP22_XLG_CTRL3_MACMODESELECT_GMAC (0 << 13)
376#define MVPP22_XLG_CTRL3_MACMODESELECT_10G (1 << 13)
Thomas Petazzoni26975822017-03-07 16:53:14 +0100377
Antoine Ténart77321952017-08-22 19:08:25 +0200378#define MVPP22_XLG_CTRL4_REG 0x184
379#define MVPP22_XLG_CTRL4_FWD_FC BIT(5)
380#define MVPP22_XLG_CTRL4_FWD_PFC BIT(6)
381#define MVPP22_XLG_CTRL4_MACMODSELECT_GMAC BIT(12)
382
Thomas Petazzoni26975822017-03-07 16:53:14 +0100383/* SMI registers. PPv2.2 only, relative to priv->iface_base. */
384#define MVPP22_SMI_MISC_CFG_REG 0x1204
Antoine Ténart81b66302017-08-22 19:08:21 +0200385#define MVPP22_SMI_POLLING_EN BIT(10)
Marcin Wojtas3f518502014-07-10 16:52:13 -0300386
Thomas Petazzonia7868412017-03-07 16:53:13 +0100387#define MVPP22_GMAC_BASE(port) (0x7000 + (port) * 0x1000 + 0xe00)
388
Marcin Wojtas3f518502014-07-10 16:52:13 -0300389#define MVPP2_CAUSE_TXQ_SENT_DESC_ALL_MASK 0xff
390
391/* Descriptor ring Macros */
392#define MVPP2_QUEUE_NEXT_DESC(q, index) \
393 (((index) < (q)->last_desc) ? ((index) + 1) : 0)
394
Antoine Ténartf84bf382017-08-22 19:08:27 +0200395/* XPCS registers. PPv2.2 only */
396#define MVPP22_MPCS_BASE(port) (0x7000 + (port) * 0x1000)
397#define MVPP22_MPCS_CTRL 0x14
398#define MVPP22_MPCS_CTRL_FWD_ERR_CONN BIT(10)
399#define MVPP22_MPCS_CLK_RESET 0x14c
400#define MAC_CLK_RESET_SD_TX BIT(0)
401#define MAC_CLK_RESET_SD_RX BIT(1)
402#define MAC_CLK_RESET_MAC BIT(2)
403#define MVPP22_MPCS_CLK_RESET_DIV_RATIO(n) ((n) << 4)
404#define MVPP22_MPCS_CLK_RESET_DIV_SET BIT(11)
405
406/* XPCS registers. PPv2.2 only */
407#define MVPP22_XPCS_BASE(port) (0x7400 + (port) * 0x1000)
408#define MVPP22_XPCS_CFG0 0x0
409#define MVPP22_XPCS_CFG0_PCS_MODE(n) ((n) << 3)
410#define MVPP22_XPCS_CFG0_ACTIVE_LANE(n) ((n) << 5)
411
412/* System controller registers. Accessed through a regmap. */
413#define GENCONF_SOFT_RESET1 0x1108
414#define GENCONF_SOFT_RESET1_GOP BIT(6)
415#define GENCONF_PORT_CTRL0 0x1110
416#define GENCONF_PORT_CTRL0_BUS_WIDTH_SELECT BIT(1)
417#define GENCONF_PORT_CTRL0_RX_DATA_SAMPLE BIT(29)
418#define GENCONF_PORT_CTRL0_CLK_DIV_PHASE_CLR BIT(31)
419#define GENCONF_PORT_CTRL1 0x1114
420#define GENCONF_PORT_CTRL1_EN(p) BIT(p)
421#define GENCONF_PORT_CTRL1_RESET(p) (BIT(p) << 28)
422#define GENCONF_CTRL0 0x1120
423#define GENCONF_CTRL0_PORT0_RGMII BIT(0)
424#define GENCONF_CTRL0_PORT1_RGMII_MII BIT(1)
425#define GENCONF_CTRL0_PORT1_RGMII BIT(2)
426
Marcin Wojtas3f518502014-07-10 16:52:13 -0300427/* Various constants */
428
429/* Coalescing */
430#define MVPP2_TXDONE_COAL_PKTS_THRESH 15
Marcin Wojtasedc660f2015-08-06 19:00:30 +0200431#define MVPP2_TXDONE_HRTIMER_PERIOD_NS 1000000UL
Thomas Petazzoni213f4282017-08-03 10:42:00 +0200432#define MVPP2_TXDONE_COAL_USEC 1000
Marcin Wojtas3f518502014-07-10 16:52:13 -0300433#define MVPP2_RX_COAL_PKTS 32
434#define MVPP2_RX_COAL_USEC 100
435
436/* The two bytes Marvell header. Either contains a special value used
437 * by Marvell switches when a specific hardware mode is enabled (not
438 * supported by this driver) or is filled automatically by zeroes on
439 * the RX side. Those two bytes being at the front of the Ethernet
440 * header, they allow to have the IP header aligned on a 4 bytes
441 * boundary automatically: the hardware skips those two bytes on its
442 * own.
443 */
444#define MVPP2_MH_SIZE 2
445#define MVPP2_ETH_TYPE_LEN 2
446#define MVPP2_PPPOE_HDR_SIZE 8
447#define MVPP2_VLAN_TAG_LEN 4
448
449/* Lbtd 802.3 type */
450#define MVPP2_IP_LBDT_TYPE 0xfffa
451
Marcin Wojtas3f518502014-07-10 16:52:13 -0300452#define MVPP2_TX_CSUM_MAX_SIZE 9800
453
454/* Timeout constants */
455#define MVPP2_TX_DISABLE_TIMEOUT_MSEC 1000
456#define MVPP2_TX_PENDING_TIMEOUT_MSEC 1000
457
458#define MVPP2_TX_MTU_MAX 0x7ffff
459
460/* Maximum number of T-CONTs of PON port */
461#define MVPP2_MAX_TCONT 16
462
463/* Maximum number of supported ports */
464#define MVPP2_MAX_PORTS 4
465
466/* Maximum number of TXQs used by single port */
467#define MVPP2_MAX_TXQ 8
468
Marcin Wojtas3f518502014-07-10 16:52:13 -0300469/* Dfault number of RXQs in use */
470#define MVPP2_DEFAULT_RXQ 4
471
Marcin Wojtas3f518502014-07-10 16:52:13 -0300472/* Max number of Rx descriptors */
473#define MVPP2_MAX_RXD 128
474
475/* Max number of Tx descriptors */
476#define MVPP2_MAX_TXD 1024
477
478/* Amount of Tx descriptors that can be reserved at once by CPU */
479#define MVPP2_CPU_DESC_CHUNK 64
480
481/* Max number of Tx descriptors in each aggregated queue */
482#define MVPP2_AGGR_TXQ_SIZE 256
483
484/* Descriptor aligned size */
485#define MVPP2_DESC_ALIGNED_SIZE 32
486
487/* Descriptor alignment mask */
488#define MVPP2_TX_DESC_ALIGN (MVPP2_DESC_ALIGNED_SIZE - 1)
489
490/* RX FIFO constants */
491#define MVPP2_RX_FIFO_PORT_DATA_SIZE 0x2000
492#define MVPP2_RX_FIFO_PORT_ATTR_SIZE 0x80
493#define MVPP2_RX_FIFO_PORT_MIN_PKT 0x80
494
495/* RX buffer constants */
496#define MVPP2_SKB_SHINFO_SIZE \
497 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
498
499#define MVPP2_RX_PKT_SIZE(mtu) \
500 ALIGN((mtu) + MVPP2_MH_SIZE + MVPP2_VLAN_TAG_LEN + \
Jisheng Zhang4a0a12d2016-04-01 17:11:05 +0800501 ETH_HLEN + ETH_FCS_LEN, cache_line_size())
Marcin Wojtas3f518502014-07-10 16:52:13 -0300502
503#define MVPP2_RX_BUF_SIZE(pkt_size) ((pkt_size) + NET_SKB_PAD)
504#define MVPP2_RX_TOTAL_SIZE(buf_size) ((buf_size) + MVPP2_SKB_SHINFO_SIZE)
505#define MVPP2_RX_MAX_PKT_SIZE(total_size) \
506 ((total_size) - NET_SKB_PAD - MVPP2_SKB_SHINFO_SIZE)
507
508#define MVPP2_BIT_TO_BYTE(bit) ((bit) / 8)
509
510/* IPv6 max L3 address size */
511#define MVPP2_MAX_L3_ADDR_SIZE 16
512
513/* Port flags */
514#define MVPP2_F_LOOPBACK BIT(0)
515
516/* Marvell tag types */
517enum mvpp2_tag_type {
518 MVPP2_TAG_TYPE_NONE = 0,
519 MVPP2_TAG_TYPE_MH = 1,
520 MVPP2_TAG_TYPE_DSA = 2,
521 MVPP2_TAG_TYPE_EDSA = 3,
522 MVPP2_TAG_TYPE_VLAN = 4,
523 MVPP2_TAG_TYPE_LAST = 5
524};
525
526/* Parser constants */
527#define MVPP2_PRS_TCAM_SRAM_SIZE 256
528#define MVPP2_PRS_TCAM_WORDS 6
529#define MVPP2_PRS_SRAM_WORDS 4
530#define MVPP2_PRS_FLOW_ID_SIZE 64
531#define MVPP2_PRS_FLOW_ID_MASK 0x3f
532#define MVPP2_PRS_TCAM_ENTRY_INVALID 1
533#define MVPP2_PRS_TCAM_DSA_TAGGED_BIT BIT(5)
534#define MVPP2_PRS_IPV4_HEAD 0x40
535#define MVPP2_PRS_IPV4_HEAD_MASK 0xf0
536#define MVPP2_PRS_IPV4_MC 0xe0
537#define MVPP2_PRS_IPV4_MC_MASK 0xf0
538#define MVPP2_PRS_IPV4_BC_MASK 0xff
539#define MVPP2_PRS_IPV4_IHL 0x5
540#define MVPP2_PRS_IPV4_IHL_MASK 0xf
541#define MVPP2_PRS_IPV6_MC 0xff
542#define MVPP2_PRS_IPV6_MC_MASK 0xff
543#define MVPP2_PRS_IPV6_HOP_MASK 0xff
544#define MVPP2_PRS_TCAM_PROTO_MASK 0xff
545#define MVPP2_PRS_TCAM_PROTO_MASK_L 0x3f
546#define MVPP2_PRS_DBL_VLANS_MAX 100
547
548/* Tcam structure:
549 * - lookup ID - 4 bits
550 * - port ID - 1 byte
551 * - additional information - 1 byte
552 * - header data - 8 bytes
553 * The fields are represented by MVPP2_PRS_TCAM_DATA_REG(5)->(0).
554 */
555#define MVPP2_PRS_AI_BITS 8
556#define MVPP2_PRS_PORT_MASK 0xff
557#define MVPP2_PRS_LU_MASK 0xf
558#define MVPP2_PRS_TCAM_DATA_BYTE(offs) \
559 (((offs) - ((offs) % 2)) * 2 + ((offs) % 2))
560#define MVPP2_PRS_TCAM_DATA_BYTE_EN(offs) \
561 (((offs) * 2) - ((offs) % 2) + 2)
562#define MVPP2_PRS_TCAM_AI_BYTE 16
563#define MVPP2_PRS_TCAM_PORT_BYTE 17
564#define MVPP2_PRS_TCAM_LU_BYTE 20
565#define MVPP2_PRS_TCAM_EN_OFFS(offs) ((offs) + 2)
566#define MVPP2_PRS_TCAM_INV_WORD 5
567/* Tcam entries ID */
568#define MVPP2_PE_DROP_ALL 0
569#define MVPP2_PE_FIRST_FREE_TID 1
570#define MVPP2_PE_LAST_FREE_TID (MVPP2_PRS_TCAM_SRAM_SIZE - 31)
571#define MVPP2_PE_IP6_EXT_PROTO_UN (MVPP2_PRS_TCAM_SRAM_SIZE - 30)
572#define MVPP2_PE_MAC_MC_IP6 (MVPP2_PRS_TCAM_SRAM_SIZE - 29)
573#define MVPP2_PE_IP6_ADDR_UN (MVPP2_PRS_TCAM_SRAM_SIZE - 28)
574#define MVPP2_PE_IP4_ADDR_UN (MVPP2_PRS_TCAM_SRAM_SIZE - 27)
575#define MVPP2_PE_LAST_DEFAULT_FLOW (MVPP2_PRS_TCAM_SRAM_SIZE - 26)
576#define MVPP2_PE_FIRST_DEFAULT_FLOW (MVPP2_PRS_TCAM_SRAM_SIZE - 19)
577#define MVPP2_PE_EDSA_TAGGED (MVPP2_PRS_TCAM_SRAM_SIZE - 18)
578#define MVPP2_PE_EDSA_UNTAGGED (MVPP2_PRS_TCAM_SRAM_SIZE - 17)
579#define MVPP2_PE_DSA_TAGGED (MVPP2_PRS_TCAM_SRAM_SIZE - 16)
580#define MVPP2_PE_DSA_UNTAGGED (MVPP2_PRS_TCAM_SRAM_SIZE - 15)
581#define MVPP2_PE_ETYPE_EDSA_TAGGED (MVPP2_PRS_TCAM_SRAM_SIZE - 14)
582#define MVPP2_PE_ETYPE_EDSA_UNTAGGED (MVPP2_PRS_TCAM_SRAM_SIZE - 13)
583#define MVPP2_PE_ETYPE_DSA_TAGGED (MVPP2_PRS_TCAM_SRAM_SIZE - 12)
584#define MVPP2_PE_ETYPE_DSA_UNTAGGED (MVPP2_PRS_TCAM_SRAM_SIZE - 11)
585#define MVPP2_PE_MH_DEFAULT (MVPP2_PRS_TCAM_SRAM_SIZE - 10)
586#define MVPP2_PE_DSA_DEFAULT (MVPP2_PRS_TCAM_SRAM_SIZE - 9)
587#define MVPP2_PE_IP6_PROTO_UN (MVPP2_PRS_TCAM_SRAM_SIZE - 8)
588#define MVPP2_PE_IP4_PROTO_UN (MVPP2_PRS_TCAM_SRAM_SIZE - 7)
589#define MVPP2_PE_ETH_TYPE_UN (MVPP2_PRS_TCAM_SRAM_SIZE - 6)
590#define MVPP2_PE_VLAN_DBL (MVPP2_PRS_TCAM_SRAM_SIZE - 5)
591#define MVPP2_PE_VLAN_NONE (MVPP2_PRS_TCAM_SRAM_SIZE - 4)
592#define MVPP2_PE_MAC_MC_ALL (MVPP2_PRS_TCAM_SRAM_SIZE - 3)
593#define MVPP2_PE_MAC_PROMISCUOUS (MVPP2_PRS_TCAM_SRAM_SIZE - 2)
594#define MVPP2_PE_MAC_NON_PROMISCUOUS (MVPP2_PRS_TCAM_SRAM_SIZE - 1)
595
596/* Sram structure
597 * The fields are represented by MVPP2_PRS_TCAM_DATA_REG(3)->(0).
598 */
599#define MVPP2_PRS_SRAM_RI_OFFS 0
600#define MVPP2_PRS_SRAM_RI_WORD 0
601#define MVPP2_PRS_SRAM_RI_CTRL_OFFS 32
602#define MVPP2_PRS_SRAM_RI_CTRL_WORD 1
603#define MVPP2_PRS_SRAM_RI_CTRL_BITS 32
604#define MVPP2_PRS_SRAM_SHIFT_OFFS 64
605#define MVPP2_PRS_SRAM_SHIFT_SIGN_BIT 72
606#define MVPP2_PRS_SRAM_UDF_OFFS 73
607#define MVPP2_PRS_SRAM_UDF_BITS 8
608#define MVPP2_PRS_SRAM_UDF_MASK 0xff
609#define MVPP2_PRS_SRAM_UDF_SIGN_BIT 81
610#define MVPP2_PRS_SRAM_UDF_TYPE_OFFS 82
611#define MVPP2_PRS_SRAM_UDF_TYPE_MASK 0x7
612#define MVPP2_PRS_SRAM_UDF_TYPE_L3 1
613#define MVPP2_PRS_SRAM_UDF_TYPE_L4 4
614#define MVPP2_PRS_SRAM_OP_SEL_SHIFT_OFFS 85
615#define MVPP2_PRS_SRAM_OP_SEL_SHIFT_MASK 0x3
616#define MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD 1
617#define MVPP2_PRS_SRAM_OP_SEL_SHIFT_IP4_ADD 2
618#define MVPP2_PRS_SRAM_OP_SEL_SHIFT_IP6_ADD 3
619#define MVPP2_PRS_SRAM_OP_SEL_UDF_OFFS 87
620#define MVPP2_PRS_SRAM_OP_SEL_UDF_BITS 2
621#define MVPP2_PRS_SRAM_OP_SEL_UDF_MASK 0x3
622#define MVPP2_PRS_SRAM_OP_SEL_UDF_ADD 0
623#define MVPP2_PRS_SRAM_OP_SEL_UDF_IP4_ADD 2
624#define MVPP2_PRS_SRAM_OP_SEL_UDF_IP6_ADD 3
625#define MVPP2_PRS_SRAM_OP_SEL_BASE_OFFS 89
626#define MVPP2_PRS_SRAM_AI_OFFS 90
627#define MVPP2_PRS_SRAM_AI_CTRL_OFFS 98
628#define MVPP2_PRS_SRAM_AI_CTRL_BITS 8
629#define MVPP2_PRS_SRAM_AI_MASK 0xff
630#define MVPP2_PRS_SRAM_NEXT_LU_OFFS 106
631#define MVPP2_PRS_SRAM_NEXT_LU_MASK 0xf
632#define MVPP2_PRS_SRAM_LU_DONE_BIT 110
633#define MVPP2_PRS_SRAM_LU_GEN_BIT 111
634
635/* Sram result info bits assignment */
636#define MVPP2_PRS_RI_MAC_ME_MASK 0x1
637#define MVPP2_PRS_RI_DSA_MASK 0x2
Thomas Petazzoni8138aff2017-02-21 11:28:11 +0100638#define MVPP2_PRS_RI_VLAN_MASK (BIT(2) | BIT(3))
639#define MVPP2_PRS_RI_VLAN_NONE 0x0
Marcin Wojtas3f518502014-07-10 16:52:13 -0300640#define MVPP2_PRS_RI_VLAN_SINGLE BIT(2)
641#define MVPP2_PRS_RI_VLAN_DOUBLE BIT(3)
642#define MVPP2_PRS_RI_VLAN_TRIPLE (BIT(2) | BIT(3))
643#define MVPP2_PRS_RI_CPU_CODE_MASK 0x70
644#define MVPP2_PRS_RI_CPU_CODE_RX_SPEC BIT(4)
Thomas Petazzoni8138aff2017-02-21 11:28:11 +0100645#define MVPP2_PRS_RI_L2_CAST_MASK (BIT(9) | BIT(10))
646#define MVPP2_PRS_RI_L2_UCAST 0x0
Marcin Wojtas3f518502014-07-10 16:52:13 -0300647#define MVPP2_PRS_RI_L2_MCAST BIT(9)
648#define MVPP2_PRS_RI_L2_BCAST BIT(10)
649#define MVPP2_PRS_RI_PPPOE_MASK 0x800
Thomas Petazzoni8138aff2017-02-21 11:28:11 +0100650#define MVPP2_PRS_RI_L3_PROTO_MASK (BIT(12) | BIT(13) | BIT(14))
651#define MVPP2_PRS_RI_L3_UN 0x0
Marcin Wojtas3f518502014-07-10 16:52:13 -0300652#define MVPP2_PRS_RI_L3_IP4 BIT(12)
653#define MVPP2_PRS_RI_L3_IP4_OPT BIT(13)
654#define MVPP2_PRS_RI_L3_IP4_OTHER (BIT(12) | BIT(13))
655#define MVPP2_PRS_RI_L3_IP6 BIT(14)
656#define MVPP2_PRS_RI_L3_IP6_EXT (BIT(12) | BIT(14))
657#define MVPP2_PRS_RI_L3_ARP (BIT(13) | BIT(14))
Thomas Petazzoni8138aff2017-02-21 11:28:11 +0100658#define MVPP2_PRS_RI_L3_ADDR_MASK (BIT(15) | BIT(16))
659#define MVPP2_PRS_RI_L3_UCAST 0x0
Marcin Wojtas3f518502014-07-10 16:52:13 -0300660#define MVPP2_PRS_RI_L3_MCAST BIT(15)
661#define MVPP2_PRS_RI_L3_BCAST (BIT(15) | BIT(16))
662#define MVPP2_PRS_RI_IP_FRAG_MASK 0x20000
663#define MVPP2_PRS_RI_UDF3_MASK 0x300000
664#define MVPP2_PRS_RI_UDF3_RX_SPECIAL BIT(21)
665#define MVPP2_PRS_RI_L4_PROTO_MASK 0x1c00000
666#define MVPP2_PRS_RI_L4_TCP BIT(22)
667#define MVPP2_PRS_RI_L4_UDP BIT(23)
668#define MVPP2_PRS_RI_L4_OTHER (BIT(22) | BIT(23))
669#define MVPP2_PRS_RI_UDF7_MASK 0x60000000
670#define MVPP2_PRS_RI_UDF7_IP6_LITE BIT(29)
671#define MVPP2_PRS_RI_DROP_MASK 0x80000000
672
673/* Sram additional info bits assignment */
674#define MVPP2_PRS_IPV4_DIP_AI_BIT BIT(0)
675#define MVPP2_PRS_IPV6_NO_EXT_AI_BIT BIT(0)
676#define MVPP2_PRS_IPV6_EXT_AI_BIT BIT(1)
677#define MVPP2_PRS_IPV6_EXT_AH_AI_BIT BIT(2)
678#define MVPP2_PRS_IPV6_EXT_AH_LEN_AI_BIT BIT(3)
679#define MVPP2_PRS_IPV6_EXT_AH_L4_AI_BIT BIT(4)
680#define MVPP2_PRS_SINGLE_VLAN_AI 0
681#define MVPP2_PRS_DBL_VLAN_AI_BIT BIT(7)
682
683/* DSA/EDSA type */
684#define MVPP2_PRS_TAGGED true
685#define MVPP2_PRS_UNTAGGED false
686#define MVPP2_PRS_EDSA true
687#define MVPP2_PRS_DSA false
688
689/* MAC entries, shadow udf */
690enum mvpp2_prs_udf {
691 MVPP2_PRS_UDF_MAC_DEF,
692 MVPP2_PRS_UDF_MAC_RANGE,
693 MVPP2_PRS_UDF_L2_DEF,
694 MVPP2_PRS_UDF_L2_DEF_COPY,
695 MVPP2_PRS_UDF_L2_USER,
696};
697
698/* Lookup ID */
699enum mvpp2_prs_lookup {
700 MVPP2_PRS_LU_MH,
701 MVPP2_PRS_LU_MAC,
702 MVPP2_PRS_LU_DSA,
703 MVPP2_PRS_LU_VLAN,
704 MVPP2_PRS_LU_L2,
705 MVPP2_PRS_LU_PPPOE,
706 MVPP2_PRS_LU_IP4,
707 MVPP2_PRS_LU_IP6,
708 MVPP2_PRS_LU_FLOWS,
709 MVPP2_PRS_LU_LAST,
710};
711
712/* L3 cast enum */
713enum mvpp2_prs_l3_cast {
714 MVPP2_PRS_L3_UNI_CAST,
715 MVPP2_PRS_L3_MULTI_CAST,
716 MVPP2_PRS_L3_BROAD_CAST
717};
718
719/* Classifier constants */
720#define MVPP2_CLS_FLOWS_TBL_SIZE 512
721#define MVPP2_CLS_FLOWS_TBL_DATA_WORDS 3
722#define MVPP2_CLS_LKP_TBL_SIZE 64
723
724/* BM constants */
725#define MVPP2_BM_POOLS_NUM 8
726#define MVPP2_BM_LONG_BUF_NUM 1024
727#define MVPP2_BM_SHORT_BUF_NUM 2048
728#define MVPP2_BM_POOL_SIZE_MAX (16*1024 - MVPP2_BM_POOL_PTR_ALIGN/4)
729#define MVPP2_BM_POOL_PTR_ALIGN 128
730#define MVPP2_BM_SWF_LONG_POOL(port) ((port > 2) ? 2 : port)
731#define MVPP2_BM_SWF_SHORT_POOL 3
732
733/* BM cookie (32 bits) definition */
734#define MVPP2_BM_COOKIE_POOL_OFFS 8
735#define MVPP2_BM_COOKIE_CPU_OFFS 24
736
737/* BM short pool packet size
738 * These value assure that for SWF the total number
739 * of bytes allocated for each buffer will be 512
740 */
741#define MVPP2_BM_SHORT_PKT_SIZE MVPP2_RX_MAX_PKT_SIZE(512)
742
Thomas Petazzonia7868412017-03-07 16:53:13 +0100743#define MVPP21_ADDR_SPACE_SZ 0
744#define MVPP22_ADDR_SPACE_SZ SZ_64K
745
Thomas Petazzonidf089aa2017-08-03 10:41:58 +0200746#define MVPP2_MAX_THREADS 8
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +0200747#define MVPP2_MAX_QVECS MVPP2_MAX_THREADS
Thomas Petazzonia7868412017-03-07 16:53:13 +0100748
Marcin Wojtas3f518502014-07-10 16:52:13 -0300749enum mvpp2_bm_type {
750 MVPP2_BM_FREE,
751 MVPP2_BM_SWF_LONG,
752 MVPP2_BM_SWF_SHORT
753};
754
755/* Definitions */
756
757/* Shared Packet Processor resources */
758struct mvpp2 {
759 /* Shared registers' base addresses */
Marcin Wojtas3f518502014-07-10 16:52:13 -0300760 void __iomem *lms_base;
Thomas Petazzonia7868412017-03-07 16:53:13 +0100761 void __iomem *iface_base;
762
Thomas Petazzonidf089aa2017-08-03 10:41:58 +0200763 /* On PPv2.2, each "software thread" can access the base
764 * register through a separate address space, each 64 KB apart
765 * from each other. Typically, such address spaces will be
766 * used per CPU.
Thomas Petazzonia7868412017-03-07 16:53:13 +0100767 */
Thomas Petazzonidf089aa2017-08-03 10:41:58 +0200768 void __iomem *swth_base[MVPP2_MAX_THREADS];
Marcin Wojtas3f518502014-07-10 16:52:13 -0300769
Antoine Ténartf84bf382017-08-22 19:08:27 +0200770 /* On PPv2.2, some port control registers are located into the system
771 * controller space. These registers are accessible through a regmap.
772 */
773 struct regmap *sysctrl_base;
774
Marcin Wojtas3f518502014-07-10 16:52:13 -0300775 /* Common clocks */
776 struct clk *pp_clk;
777 struct clk *gop_clk;
Thomas Petazzonifceb55d2017-03-07 16:53:18 +0100778 struct clk *mg_clk;
Marcin Wojtas3f518502014-07-10 16:52:13 -0300779
780 /* List of pointers to port structures */
781 struct mvpp2_port **port_list;
782
783 /* Aggregated TXQs */
784 struct mvpp2_tx_queue *aggr_txqs;
785
786 /* BM pools */
787 struct mvpp2_bm_pool *bm_pools;
788
789 /* PRS shadow table */
790 struct mvpp2_prs_shadow *prs_shadow;
791 /* PRS auxiliary table for double vlan entries control */
792 bool *prs_double_vlans;
793
794 /* Tclk value */
795 u32 tclk;
Thomas Petazzonifaca9242017-03-07 16:53:06 +0100796
797 /* HW version */
798 enum { MVPP21, MVPP22 } hw_version;
Thomas Petazzoni59b9a312017-03-07 16:53:17 +0100799
800 /* Maximum number of RXQs per port */
801 unsigned int max_port_rxqs;
Marcin Wojtas3f518502014-07-10 16:52:13 -0300802};
803
804struct mvpp2_pcpu_stats {
805 struct u64_stats_sync syncp;
806 u64 rx_packets;
807 u64 rx_bytes;
808 u64 tx_packets;
809 u64 tx_bytes;
810};
811
Marcin Wojtasedc660f2015-08-06 19:00:30 +0200812/* Per-CPU port control */
813struct mvpp2_port_pcpu {
814 struct hrtimer tx_done_timer;
815 bool timer_scheduled;
816 /* Tasklet for egress finalization */
817 struct tasklet_struct tx_done_tasklet;
818};
819
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +0200820struct mvpp2_queue_vector {
821 int irq;
822 struct napi_struct napi;
823 enum { MVPP2_QUEUE_VECTOR_SHARED, MVPP2_QUEUE_VECTOR_PRIVATE } type;
824 int sw_thread_id;
825 u16 sw_thread_mask;
826 int first_rxq;
827 int nrxqs;
828 u32 pending_cause_rx;
829 struct mvpp2_port *port;
830};
831
Marcin Wojtas3f518502014-07-10 16:52:13 -0300832struct mvpp2_port {
833 u8 id;
834
Thomas Petazzonia7868412017-03-07 16:53:13 +0100835 /* Index of the port from the "group of ports" complex point
836 * of view
837 */
838 int gop_id;
839
Marcin Wojtas3f518502014-07-10 16:52:13 -0300840 struct mvpp2 *priv;
841
842 /* Per-port registers' base address */
843 void __iomem *base;
844
845 struct mvpp2_rx_queue **rxqs;
Thomas Petazzoni09f83972017-08-03 10:41:57 +0200846 unsigned int nrxqs;
Marcin Wojtas3f518502014-07-10 16:52:13 -0300847 struct mvpp2_tx_queue **txqs;
Thomas Petazzoni09f83972017-08-03 10:41:57 +0200848 unsigned int ntxqs;
Marcin Wojtas3f518502014-07-10 16:52:13 -0300849 struct net_device *dev;
850
851 int pkt_size;
852
Marcin Wojtasedc660f2015-08-06 19:00:30 +0200853 /* Per-CPU port control */
854 struct mvpp2_port_pcpu __percpu *pcpu;
855
Marcin Wojtas3f518502014-07-10 16:52:13 -0300856 /* Flags */
857 unsigned long flags;
858
859 u16 tx_ring_size;
860 u16 rx_ring_size;
861 struct mvpp2_pcpu_stats __percpu *stats;
862
Marcin Wojtas3f518502014-07-10 16:52:13 -0300863 phy_interface_t phy_interface;
864 struct device_node *phy_node;
Antoine Tenart542897d2017-08-30 10:29:15 +0200865 struct phy *comphy;
Marcin Wojtas3f518502014-07-10 16:52:13 -0300866 unsigned int link;
867 unsigned int duplex;
868 unsigned int speed;
869
870 struct mvpp2_bm_pool *pool_long;
871 struct mvpp2_bm_pool *pool_short;
872
873 /* Index of first port's physical RXQ */
874 u8 first_rxq;
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +0200875
876 struct mvpp2_queue_vector qvecs[MVPP2_MAX_QVECS];
877 unsigned int nqvecs;
Thomas Petazzoni213f4282017-08-03 10:42:00 +0200878 bool has_tx_irqs;
879
880 u32 tx_time_coal;
Marcin Wojtas3f518502014-07-10 16:52:13 -0300881};
882
883/* The mvpp2_tx_desc and mvpp2_rx_desc structures describe the
884 * layout of the transmit and reception DMA descriptors, and their
885 * layout is therefore defined by the hardware design
886 */
887
888#define MVPP2_TXD_L3_OFF_SHIFT 0
889#define MVPP2_TXD_IP_HLEN_SHIFT 8
890#define MVPP2_TXD_L4_CSUM_FRAG BIT(13)
891#define MVPP2_TXD_L4_CSUM_NOT BIT(14)
892#define MVPP2_TXD_IP_CSUM_DISABLE BIT(15)
893#define MVPP2_TXD_PADDING_DISABLE BIT(23)
894#define MVPP2_TXD_L4_UDP BIT(24)
895#define MVPP2_TXD_L3_IP6 BIT(26)
896#define MVPP2_TXD_L_DESC BIT(28)
897#define MVPP2_TXD_F_DESC BIT(29)
898
899#define MVPP2_RXD_ERR_SUMMARY BIT(15)
900#define MVPP2_RXD_ERR_CODE_MASK (BIT(13) | BIT(14))
901#define MVPP2_RXD_ERR_CRC 0x0
902#define MVPP2_RXD_ERR_OVERRUN BIT(13)
903#define MVPP2_RXD_ERR_RESOURCE (BIT(13) | BIT(14))
904#define MVPP2_RXD_BM_POOL_ID_OFFS 16
905#define MVPP2_RXD_BM_POOL_ID_MASK (BIT(16) | BIT(17) | BIT(18))
906#define MVPP2_RXD_HWF_SYNC BIT(21)
907#define MVPP2_RXD_L4_CSUM_OK BIT(22)
908#define MVPP2_RXD_IP4_HEADER_ERR BIT(24)
909#define MVPP2_RXD_L4_TCP BIT(25)
910#define MVPP2_RXD_L4_UDP BIT(26)
911#define MVPP2_RXD_L3_IP4 BIT(28)
912#define MVPP2_RXD_L3_IP6 BIT(30)
913#define MVPP2_RXD_BUF_HDR BIT(31)
914
Thomas Petazzoni054f6372017-03-07 16:53:07 +0100915/* HW TX descriptor for PPv2.1 */
916struct mvpp21_tx_desc {
Marcin Wojtas3f518502014-07-10 16:52:13 -0300917 u32 command; /* Options used by HW for packet transmitting.*/
918 u8 packet_offset; /* the offset from the buffer beginning */
919 u8 phys_txq; /* destination queue ID */
920 u16 data_size; /* data size of transmitted packet in bytes */
Thomas Petazzoni20396132017-03-07 16:53:00 +0100921 u32 buf_dma_addr; /* physical addr of transmitted buffer */
Marcin Wojtas3f518502014-07-10 16:52:13 -0300922 u32 buf_cookie; /* cookie for access to TX buffer in tx path */
923 u32 reserved1[3]; /* hw_cmd (for future use, BM, PON, PNC) */
924 u32 reserved2; /* reserved (for future use) */
925};
926
Thomas Petazzoni054f6372017-03-07 16:53:07 +0100927/* HW RX descriptor for PPv2.1 */
928struct mvpp21_rx_desc {
Marcin Wojtas3f518502014-07-10 16:52:13 -0300929 u32 status; /* info about received packet */
930 u16 reserved1; /* parser_info (for future use, PnC) */
931 u16 data_size; /* size of received packet in bytes */
Thomas Petazzoni20396132017-03-07 16:53:00 +0100932 u32 buf_dma_addr; /* physical address of the buffer */
Marcin Wojtas3f518502014-07-10 16:52:13 -0300933 u32 buf_cookie; /* cookie for access to RX buffer in rx path */
934 u16 reserved2; /* gem_port_id (for future use, PON) */
935 u16 reserved3; /* csum_l4 (for future use, PnC) */
936 u8 reserved4; /* bm_qset (for future use, BM) */
937 u8 reserved5;
938 u16 reserved6; /* classify_info (for future use, PnC) */
939 u32 reserved7; /* flow_id (for future use, PnC) */
940 u32 reserved8;
941};
942
Thomas Petazzonie7c53592017-03-07 16:53:08 +0100943/* HW TX descriptor for PPv2.2 */
944struct mvpp22_tx_desc {
945 u32 command;
946 u8 packet_offset;
947 u8 phys_txq;
948 u16 data_size;
949 u64 reserved1;
950 u64 buf_dma_addr_ptp;
951 u64 buf_cookie_misc;
952};
953
954/* HW RX descriptor for PPv2.2 */
955struct mvpp22_rx_desc {
956 u32 status;
957 u16 reserved1;
958 u16 data_size;
959 u32 reserved2;
960 u32 reserved3;
961 u64 buf_dma_addr_key_hash;
962 u64 buf_cookie_misc;
963};
964
Thomas Petazzoni054f6372017-03-07 16:53:07 +0100965/* Opaque type used by the driver to manipulate the HW TX and RX
966 * descriptors
967 */
968struct mvpp2_tx_desc {
969 union {
970 struct mvpp21_tx_desc pp21;
Thomas Petazzonie7c53592017-03-07 16:53:08 +0100971 struct mvpp22_tx_desc pp22;
Thomas Petazzoni054f6372017-03-07 16:53:07 +0100972 };
973};
974
975struct mvpp2_rx_desc {
976 union {
977 struct mvpp21_rx_desc pp21;
Thomas Petazzonie7c53592017-03-07 16:53:08 +0100978 struct mvpp22_rx_desc pp22;
Thomas Petazzoni054f6372017-03-07 16:53:07 +0100979 };
980};
981
Thomas Petazzoni83544912016-12-21 11:28:49 +0100982struct mvpp2_txq_pcpu_buf {
983 /* Transmitted SKB */
984 struct sk_buff *skb;
985
986 /* Physical address of transmitted buffer */
Thomas Petazzoni20396132017-03-07 16:53:00 +0100987 dma_addr_t dma;
Thomas Petazzoni83544912016-12-21 11:28:49 +0100988
989 /* Size transmitted */
990 size_t size;
991};
992
Marcin Wojtas3f518502014-07-10 16:52:13 -0300993/* Per-CPU Tx queue control */
994struct mvpp2_txq_pcpu {
995 int cpu;
996
997 /* Number of Tx DMA descriptors in the descriptor ring */
998 int size;
999
1000 /* Number of currently used Tx DMA descriptor in the
1001 * descriptor ring
1002 */
1003 int count;
1004
1005 /* Number of Tx DMA descriptors reserved for each CPU */
1006 int reserved_num;
1007
Thomas Petazzoni83544912016-12-21 11:28:49 +01001008 /* Infos about transmitted buffers */
1009 struct mvpp2_txq_pcpu_buf *buffs;
Marcin Wojtas71ce3912015-08-06 19:00:29 +02001010
Marcin Wojtas3f518502014-07-10 16:52:13 -03001011 /* Index of last TX DMA descriptor that was inserted */
1012 int txq_put_index;
1013
1014 /* Index of the TX DMA descriptor to be cleaned up */
1015 int txq_get_index;
Antoine Ténart186cd4d2017-08-23 09:46:56 +02001016
1017 /* DMA buffer for TSO headers */
1018 char *tso_headers;
1019 dma_addr_t tso_headers_dma;
Marcin Wojtas3f518502014-07-10 16:52:13 -03001020};
1021
1022struct mvpp2_tx_queue {
1023 /* Physical number of this Tx queue */
1024 u8 id;
1025
1026 /* Logical number of this Tx queue */
1027 u8 log_id;
1028
1029 /* Number of Tx DMA descriptors in the descriptor ring */
1030 int size;
1031
1032 /* Number of currently used Tx DMA descriptor in the descriptor ring */
1033 int count;
1034
1035 /* Per-CPU control of physical Tx queues */
1036 struct mvpp2_txq_pcpu __percpu *pcpu;
1037
Marcin Wojtas3f518502014-07-10 16:52:13 -03001038 u32 done_pkts_coal;
1039
1040 /* Virtual address of thex Tx DMA descriptors array */
1041 struct mvpp2_tx_desc *descs;
1042
1043 /* DMA address of the Tx DMA descriptors array */
Thomas Petazzoni20396132017-03-07 16:53:00 +01001044 dma_addr_t descs_dma;
Marcin Wojtas3f518502014-07-10 16:52:13 -03001045
1046 /* Index of the last Tx DMA descriptor */
1047 int last_desc;
1048
1049 /* Index of the next Tx DMA descriptor to process */
1050 int next_desc_to_proc;
1051};
1052
1053struct mvpp2_rx_queue {
1054 /* RX queue number, in the range 0-31 for physical RXQs */
1055 u8 id;
1056
1057 /* Num of rx descriptors in the rx descriptor ring */
1058 int size;
1059
1060 u32 pkts_coal;
1061 u32 time_coal;
1062
1063 /* Virtual address of the RX DMA descriptors array */
1064 struct mvpp2_rx_desc *descs;
1065
1066 /* DMA address of the RX DMA descriptors array */
Thomas Petazzoni20396132017-03-07 16:53:00 +01001067 dma_addr_t descs_dma;
Marcin Wojtas3f518502014-07-10 16:52:13 -03001068
1069 /* Index of the last RX DMA descriptor */
1070 int last_desc;
1071
1072 /* Index of the next RX DMA descriptor to process */
1073 int next_desc_to_proc;
1074
1075 /* ID of port to which physical RXQ is mapped */
1076 int port;
1077
1078 /* Port's logic RXQ number to which physical RXQ is mapped */
1079 int logic_rxq;
1080};
1081
1082union mvpp2_prs_tcam_entry {
1083 u32 word[MVPP2_PRS_TCAM_WORDS];
1084 u8 byte[MVPP2_PRS_TCAM_WORDS * 4];
1085};
1086
1087union mvpp2_prs_sram_entry {
1088 u32 word[MVPP2_PRS_SRAM_WORDS];
1089 u8 byte[MVPP2_PRS_SRAM_WORDS * 4];
1090};
1091
1092struct mvpp2_prs_entry {
1093 u32 index;
1094 union mvpp2_prs_tcam_entry tcam;
1095 union mvpp2_prs_sram_entry sram;
1096};
1097
1098struct mvpp2_prs_shadow {
1099 bool valid;
1100 bool finish;
1101
1102 /* Lookup ID */
1103 int lu;
1104
1105 /* User defined offset */
1106 int udf;
1107
1108 /* Result info */
1109 u32 ri;
1110 u32 ri_mask;
1111};
1112
1113struct mvpp2_cls_flow_entry {
1114 u32 index;
1115 u32 data[MVPP2_CLS_FLOWS_TBL_DATA_WORDS];
1116};
1117
1118struct mvpp2_cls_lookup_entry {
1119 u32 lkpid;
1120 u32 way;
1121 u32 data;
1122};
1123
1124struct mvpp2_bm_pool {
1125 /* Pool number in the range 0-7 */
1126 int id;
1127 enum mvpp2_bm_type type;
1128
1129 /* Buffer Pointers Pool External (BPPE) size */
1130 int size;
Thomas Petazzonid01524d2017-03-07 16:53:09 +01001131 /* BPPE size in bytes */
1132 int size_bytes;
Marcin Wojtas3f518502014-07-10 16:52:13 -03001133 /* Number of buffers for this pool */
1134 int buf_num;
1135 /* Pool buffer size */
1136 int buf_size;
1137 /* Packet size */
1138 int pkt_size;
Thomas Petazzoni0e037282017-02-21 11:28:12 +01001139 int frag_size;
Marcin Wojtas3f518502014-07-10 16:52:13 -03001140
1141 /* BPPE virtual base address */
1142 u32 *virt_addr;
Thomas Petazzoni20396132017-03-07 16:53:00 +01001143 /* BPPE DMA base address */
1144 dma_addr_t dma_addr;
Marcin Wojtas3f518502014-07-10 16:52:13 -03001145
1146 /* Ports using BM pool */
1147 u32 port_map;
Marcin Wojtas3f518502014-07-10 16:52:13 -03001148};
1149
Thomas Petazzoni213f4282017-08-03 10:42:00 +02001150/* Queue modes */
1151#define MVPP2_QDIST_SINGLE_MODE 0
1152#define MVPP2_QDIST_MULTI_MODE 1
1153
1154static int queue_mode = MVPP2_QDIST_SINGLE_MODE;
1155
1156module_param(queue_mode, int, 0444);
1157MODULE_PARM_DESC(queue_mode, "Set queue_mode (single=0, multi=1)");
1158
Marcin Wojtas3f518502014-07-10 16:52:13 -03001159#define MVPP2_DRIVER_NAME "mvpp2"
1160#define MVPP2_DRIVER_VERSION "1.0"
1161
1162/* Utility/helper methods */
1163
1164static void mvpp2_write(struct mvpp2 *priv, u32 offset, u32 data)
1165{
Thomas Petazzonidf089aa2017-08-03 10:41:58 +02001166 writel(data, priv->swth_base[0] + offset);
Marcin Wojtas3f518502014-07-10 16:52:13 -03001167}
1168
1169static u32 mvpp2_read(struct mvpp2 *priv, u32 offset)
1170{
Thomas Petazzonidf089aa2017-08-03 10:41:58 +02001171 return readl(priv->swth_base[0] + offset);
Thomas Petazzonia7868412017-03-07 16:53:13 +01001172}
1173
1174/* These accessors should be used to access:
1175 *
1176 * - per-CPU registers, where each CPU has its own copy of the
1177 * register.
1178 *
1179 * MVPP2_BM_VIRT_ALLOC_REG
1180 * MVPP2_BM_ADDR_HIGH_ALLOC
1181 * MVPP22_BM_ADDR_HIGH_RLS_REG
1182 * MVPP2_BM_VIRT_RLS_REG
1183 * MVPP2_ISR_RX_TX_CAUSE_REG
1184 * MVPP2_ISR_RX_TX_MASK_REG
1185 * MVPP2_TXQ_NUM_REG
1186 * MVPP2_AGGR_TXQ_UPDATE_REG
1187 * MVPP2_TXQ_RSVD_REQ_REG
1188 * MVPP2_TXQ_RSVD_RSLT_REG
1189 * MVPP2_TXQ_SENT_REG
1190 * MVPP2_RXQ_NUM_REG
1191 *
1192 * - global registers that must be accessed through a specific CPU
1193 * window, because they are related to an access to a per-CPU
1194 * register
1195 *
1196 * MVPP2_BM_PHY_ALLOC_REG (related to MVPP2_BM_VIRT_ALLOC_REG)
1197 * MVPP2_BM_PHY_RLS_REG (related to MVPP2_BM_VIRT_RLS_REG)
1198 * MVPP2_RXQ_THRESH_REG (related to MVPP2_RXQ_NUM_REG)
1199 * MVPP2_RXQ_DESC_ADDR_REG (related to MVPP2_RXQ_NUM_REG)
1200 * MVPP2_RXQ_DESC_SIZE_REG (related to MVPP2_RXQ_NUM_REG)
1201 * MVPP2_RXQ_INDEX_REG (related to MVPP2_RXQ_NUM_REG)
1202 * MVPP2_TXQ_PENDING_REG (related to MVPP2_TXQ_NUM_REG)
1203 * MVPP2_TXQ_DESC_ADDR_REG (related to MVPP2_TXQ_NUM_REG)
1204 * MVPP2_TXQ_DESC_SIZE_REG (related to MVPP2_TXQ_NUM_REG)
1205 * MVPP2_TXQ_INDEX_REG (related to MVPP2_TXQ_NUM_REG)
1206 * MVPP2_TXQ_PENDING_REG (related to MVPP2_TXQ_NUM_REG)
1207 * MVPP2_TXQ_PREF_BUF_REG (related to MVPP2_TXQ_NUM_REG)
1208 * MVPP2_TXQ_PREF_BUF_REG (related to MVPP2_TXQ_NUM_REG)
1209 */
1210static void mvpp2_percpu_write(struct mvpp2 *priv, int cpu,
1211 u32 offset, u32 data)
1212{
Thomas Petazzonidf089aa2017-08-03 10:41:58 +02001213 writel(data, priv->swth_base[cpu] + offset);
Thomas Petazzonia7868412017-03-07 16:53:13 +01001214}
1215
1216static u32 mvpp2_percpu_read(struct mvpp2 *priv, int cpu,
1217 u32 offset)
1218{
Thomas Petazzonidf089aa2017-08-03 10:41:58 +02001219 return readl(priv->swth_base[cpu] + offset);
Marcin Wojtas3f518502014-07-10 16:52:13 -03001220}
1221
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01001222static dma_addr_t mvpp2_txdesc_dma_addr_get(struct mvpp2_port *port,
1223 struct mvpp2_tx_desc *tx_desc)
1224{
Thomas Petazzonie7c53592017-03-07 16:53:08 +01001225 if (port->priv->hw_version == MVPP21)
1226 return tx_desc->pp21.buf_dma_addr;
1227 else
1228 return tx_desc->pp22.buf_dma_addr_ptp & GENMASK_ULL(40, 0);
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01001229}
1230
1231static void mvpp2_txdesc_dma_addr_set(struct mvpp2_port *port,
1232 struct mvpp2_tx_desc *tx_desc,
1233 dma_addr_t dma_addr)
1234{
Thomas Petazzonie7c53592017-03-07 16:53:08 +01001235 if (port->priv->hw_version == MVPP21) {
1236 tx_desc->pp21.buf_dma_addr = dma_addr;
1237 } else {
1238 u64 val = (u64)dma_addr;
1239
1240 tx_desc->pp22.buf_dma_addr_ptp &= ~GENMASK_ULL(40, 0);
1241 tx_desc->pp22.buf_dma_addr_ptp |= val;
1242 }
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01001243}
1244
1245static size_t mvpp2_txdesc_size_get(struct mvpp2_port *port,
1246 struct mvpp2_tx_desc *tx_desc)
1247{
Thomas Petazzonie7c53592017-03-07 16:53:08 +01001248 if (port->priv->hw_version == MVPP21)
1249 return tx_desc->pp21.data_size;
1250 else
1251 return tx_desc->pp22.data_size;
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01001252}
1253
1254static void mvpp2_txdesc_size_set(struct mvpp2_port *port,
1255 struct mvpp2_tx_desc *tx_desc,
1256 size_t size)
1257{
Thomas Petazzonie7c53592017-03-07 16:53:08 +01001258 if (port->priv->hw_version == MVPP21)
1259 tx_desc->pp21.data_size = size;
1260 else
1261 tx_desc->pp22.data_size = size;
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01001262}
1263
1264static void mvpp2_txdesc_txq_set(struct mvpp2_port *port,
1265 struct mvpp2_tx_desc *tx_desc,
1266 unsigned int txq)
1267{
Thomas Petazzonie7c53592017-03-07 16:53:08 +01001268 if (port->priv->hw_version == MVPP21)
1269 tx_desc->pp21.phys_txq = txq;
1270 else
1271 tx_desc->pp22.phys_txq = txq;
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01001272}
1273
1274static void mvpp2_txdesc_cmd_set(struct mvpp2_port *port,
1275 struct mvpp2_tx_desc *tx_desc,
1276 unsigned int command)
1277{
Thomas Petazzonie7c53592017-03-07 16:53:08 +01001278 if (port->priv->hw_version == MVPP21)
1279 tx_desc->pp21.command = command;
1280 else
1281 tx_desc->pp22.command = command;
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01001282}
1283
1284static void mvpp2_txdesc_offset_set(struct mvpp2_port *port,
1285 struct mvpp2_tx_desc *tx_desc,
1286 unsigned int offset)
1287{
Thomas Petazzonie7c53592017-03-07 16:53:08 +01001288 if (port->priv->hw_version == MVPP21)
1289 tx_desc->pp21.packet_offset = offset;
1290 else
1291 tx_desc->pp22.packet_offset = offset;
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01001292}
1293
1294static unsigned int mvpp2_txdesc_offset_get(struct mvpp2_port *port,
1295 struct mvpp2_tx_desc *tx_desc)
1296{
Thomas Petazzonie7c53592017-03-07 16:53:08 +01001297 if (port->priv->hw_version == MVPP21)
1298 return tx_desc->pp21.packet_offset;
1299 else
1300 return tx_desc->pp22.packet_offset;
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01001301}
1302
1303static dma_addr_t mvpp2_rxdesc_dma_addr_get(struct mvpp2_port *port,
1304 struct mvpp2_rx_desc *rx_desc)
1305{
Thomas Petazzonie7c53592017-03-07 16:53:08 +01001306 if (port->priv->hw_version == MVPP21)
1307 return rx_desc->pp21.buf_dma_addr;
1308 else
1309 return rx_desc->pp22.buf_dma_addr_key_hash & GENMASK_ULL(40, 0);
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01001310}
1311
1312static unsigned long mvpp2_rxdesc_cookie_get(struct mvpp2_port *port,
1313 struct mvpp2_rx_desc *rx_desc)
1314{
Thomas Petazzonie7c53592017-03-07 16:53:08 +01001315 if (port->priv->hw_version == MVPP21)
1316 return rx_desc->pp21.buf_cookie;
1317 else
1318 return rx_desc->pp22.buf_cookie_misc & GENMASK_ULL(40, 0);
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01001319}
1320
1321static size_t mvpp2_rxdesc_size_get(struct mvpp2_port *port,
1322 struct mvpp2_rx_desc *rx_desc)
1323{
Thomas Petazzonie7c53592017-03-07 16:53:08 +01001324 if (port->priv->hw_version == MVPP21)
1325 return rx_desc->pp21.data_size;
1326 else
1327 return rx_desc->pp22.data_size;
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01001328}
1329
1330static u32 mvpp2_rxdesc_status_get(struct mvpp2_port *port,
1331 struct mvpp2_rx_desc *rx_desc)
1332{
Thomas Petazzonie7c53592017-03-07 16:53:08 +01001333 if (port->priv->hw_version == MVPP21)
1334 return rx_desc->pp21.status;
1335 else
1336 return rx_desc->pp22.status;
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01001337}
1338
Marcin Wojtas3f518502014-07-10 16:52:13 -03001339static void mvpp2_txq_inc_get(struct mvpp2_txq_pcpu *txq_pcpu)
1340{
1341 txq_pcpu->txq_get_index++;
1342 if (txq_pcpu->txq_get_index == txq_pcpu->size)
1343 txq_pcpu->txq_get_index = 0;
1344}
1345
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01001346static void mvpp2_txq_inc_put(struct mvpp2_port *port,
1347 struct mvpp2_txq_pcpu *txq_pcpu,
Marcin Wojtas71ce3912015-08-06 19:00:29 +02001348 struct sk_buff *skb,
1349 struct mvpp2_tx_desc *tx_desc)
Marcin Wojtas3f518502014-07-10 16:52:13 -03001350{
Thomas Petazzoni83544912016-12-21 11:28:49 +01001351 struct mvpp2_txq_pcpu_buf *tx_buf =
1352 txq_pcpu->buffs + txq_pcpu->txq_put_index;
1353 tx_buf->skb = skb;
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01001354 tx_buf->size = mvpp2_txdesc_size_get(port, tx_desc);
1355 tx_buf->dma = mvpp2_txdesc_dma_addr_get(port, tx_desc) +
1356 mvpp2_txdesc_offset_get(port, tx_desc);
Marcin Wojtas3f518502014-07-10 16:52:13 -03001357 txq_pcpu->txq_put_index++;
1358 if (txq_pcpu->txq_put_index == txq_pcpu->size)
1359 txq_pcpu->txq_put_index = 0;
1360}
1361
1362/* Get number of physical egress port */
1363static inline int mvpp2_egress_port(struct mvpp2_port *port)
1364{
1365 return MVPP2_MAX_TCONT + port->id;
1366}
1367
1368/* Get number of physical TXQ */
1369static inline int mvpp2_txq_phys(int port, int txq)
1370{
1371 return (MVPP2_MAX_TCONT + port) * MVPP2_MAX_TXQ + txq;
1372}
1373
1374/* Parser configuration routines */
1375
1376/* Update parser tcam and sram hw entries */
1377static int mvpp2_prs_hw_write(struct mvpp2 *priv, struct mvpp2_prs_entry *pe)
1378{
1379 int i;
1380
1381 if (pe->index > MVPP2_PRS_TCAM_SRAM_SIZE - 1)
1382 return -EINVAL;
1383
1384 /* Clear entry invalidation bit */
1385 pe->tcam.word[MVPP2_PRS_TCAM_INV_WORD] &= ~MVPP2_PRS_TCAM_INV_MASK;
1386
1387 /* Write tcam index - indirect access */
1388 mvpp2_write(priv, MVPP2_PRS_TCAM_IDX_REG, pe->index);
1389 for (i = 0; i < MVPP2_PRS_TCAM_WORDS; i++)
1390 mvpp2_write(priv, MVPP2_PRS_TCAM_DATA_REG(i), pe->tcam.word[i]);
1391
1392 /* Write sram index - indirect access */
1393 mvpp2_write(priv, MVPP2_PRS_SRAM_IDX_REG, pe->index);
1394 for (i = 0; i < MVPP2_PRS_SRAM_WORDS; i++)
1395 mvpp2_write(priv, MVPP2_PRS_SRAM_DATA_REG(i), pe->sram.word[i]);
1396
1397 return 0;
1398}
1399
1400/* Read tcam entry from hw */
1401static int mvpp2_prs_hw_read(struct mvpp2 *priv, struct mvpp2_prs_entry *pe)
1402{
1403 int i;
1404
1405 if (pe->index > MVPP2_PRS_TCAM_SRAM_SIZE - 1)
1406 return -EINVAL;
1407
1408 /* Write tcam index - indirect access */
1409 mvpp2_write(priv, MVPP2_PRS_TCAM_IDX_REG, pe->index);
1410
1411 pe->tcam.word[MVPP2_PRS_TCAM_INV_WORD] = mvpp2_read(priv,
1412 MVPP2_PRS_TCAM_DATA_REG(MVPP2_PRS_TCAM_INV_WORD));
1413 if (pe->tcam.word[MVPP2_PRS_TCAM_INV_WORD] & MVPP2_PRS_TCAM_INV_MASK)
1414 return MVPP2_PRS_TCAM_ENTRY_INVALID;
1415
1416 for (i = 0; i < MVPP2_PRS_TCAM_WORDS; i++)
1417 pe->tcam.word[i] = mvpp2_read(priv, MVPP2_PRS_TCAM_DATA_REG(i));
1418
1419 /* Write sram index - indirect access */
1420 mvpp2_write(priv, MVPP2_PRS_SRAM_IDX_REG, pe->index);
1421 for (i = 0; i < MVPP2_PRS_SRAM_WORDS; i++)
1422 pe->sram.word[i] = mvpp2_read(priv, MVPP2_PRS_SRAM_DATA_REG(i));
1423
1424 return 0;
1425}
1426
1427/* Invalidate tcam hw entry */
1428static void mvpp2_prs_hw_inv(struct mvpp2 *priv, int index)
1429{
1430 /* Write index - indirect access */
1431 mvpp2_write(priv, MVPP2_PRS_TCAM_IDX_REG, index);
1432 mvpp2_write(priv, MVPP2_PRS_TCAM_DATA_REG(MVPP2_PRS_TCAM_INV_WORD),
1433 MVPP2_PRS_TCAM_INV_MASK);
1434}
1435
1436/* Enable shadow table entry and set its lookup ID */
1437static void mvpp2_prs_shadow_set(struct mvpp2 *priv, int index, int lu)
1438{
1439 priv->prs_shadow[index].valid = true;
1440 priv->prs_shadow[index].lu = lu;
1441}
1442
1443/* Update ri fields in shadow table entry */
1444static void mvpp2_prs_shadow_ri_set(struct mvpp2 *priv, int index,
1445 unsigned int ri, unsigned int ri_mask)
1446{
1447 priv->prs_shadow[index].ri_mask = ri_mask;
1448 priv->prs_shadow[index].ri = ri;
1449}
1450
1451/* Update lookup field in tcam sw entry */
1452static void mvpp2_prs_tcam_lu_set(struct mvpp2_prs_entry *pe, unsigned int lu)
1453{
1454 int enable_off = MVPP2_PRS_TCAM_EN_OFFS(MVPP2_PRS_TCAM_LU_BYTE);
1455
1456 pe->tcam.byte[MVPP2_PRS_TCAM_LU_BYTE] = lu;
1457 pe->tcam.byte[enable_off] = MVPP2_PRS_LU_MASK;
1458}
1459
1460/* Update mask for single port in tcam sw entry */
1461static void mvpp2_prs_tcam_port_set(struct mvpp2_prs_entry *pe,
1462 unsigned int port, bool add)
1463{
1464 int enable_off = MVPP2_PRS_TCAM_EN_OFFS(MVPP2_PRS_TCAM_PORT_BYTE);
1465
1466 if (add)
1467 pe->tcam.byte[enable_off] &= ~(1 << port);
1468 else
1469 pe->tcam.byte[enable_off] |= 1 << port;
1470}
1471
1472/* Update port map in tcam sw entry */
1473static void mvpp2_prs_tcam_port_map_set(struct mvpp2_prs_entry *pe,
1474 unsigned int ports)
1475{
1476 unsigned char port_mask = MVPP2_PRS_PORT_MASK;
1477 int enable_off = MVPP2_PRS_TCAM_EN_OFFS(MVPP2_PRS_TCAM_PORT_BYTE);
1478
1479 pe->tcam.byte[MVPP2_PRS_TCAM_PORT_BYTE] = 0;
1480 pe->tcam.byte[enable_off] &= ~port_mask;
1481 pe->tcam.byte[enable_off] |= ~ports & MVPP2_PRS_PORT_MASK;
1482}
1483
1484/* Obtain port map from tcam sw entry */
1485static unsigned int mvpp2_prs_tcam_port_map_get(struct mvpp2_prs_entry *pe)
1486{
1487 int enable_off = MVPP2_PRS_TCAM_EN_OFFS(MVPP2_PRS_TCAM_PORT_BYTE);
1488
1489 return ~(pe->tcam.byte[enable_off]) & MVPP2_PRS_PORT_MASK;
1490}
1491
1492/* Set byte of data and its enable bits in tcam sw entry */
1493static void mvpp2_prs_tcam_data_byte_set(struct mvpp2_prs_entry *pe,
1494 unsigned int offs, unsigned char byte,
1495 unsigned char enable)
1496{
1497 pe->tcam.byte[MVPP2_PRS_TCAM_DATA_BYTE(offs)] = byte;
1498 pe->tcam.byte[MVPP2_PRS_TCAM_DATA_BYTE_EN(offs)] = enable;
1499}
1500
1501/* Get byte of data and its enable bits from tcam sw entry */
1502static void mvpp2_prs_tcam_data_byte_get(struct mvpp2_prs_entry *pe,
1503 unsigned int offs, unsigned char *byte,
1504 unsigned char *enable)
1505{
1506 *byte = pe->tcam.byte[MVPP2_PRS_TCAM_DATA_BYTE(offs)];
1507 *enable = pe->tcam.byte[MVPP2_PRS_TCAM_DATA_BYTE_EN(offs)];
1508}
1509
1510/* Compare tcam data bytes with a pattern */
1511static bool mvpp2_prs_tcam_data_cmp(struct mvpp2_prs_entry *pe, int offs,
1512 u16 data)
1513{
1514 int off = MVPP2_PRS_TCAM_DATA_BYTE(offs);
1515 u16 tcam_data;
1516
1517 tcam_data = (8 << pe->tcam.byte[off + 1]) | pe->tcam.byte[off];
1518 if (tcam_data != data)
1519 return false;
1520 return true;
1521}
1522
1523/* Update ai bits in tcam sw entry */
1524static void mvpp2_prs_tcam_ai_update(struct mvpp2_prs_entry *pe,
1525 unsigned int bits, unsigned int enable)
1526{
1527 int i, ai_idx = MVPP2_PRS_TCAM_AI_BYTE;
1528
1529 for (i = 0; i < MVPP2_PRS_AI_BITS; i++) {
1530
1531 if (!(enable & BIT(i)))
1532 continue;
1533
1534 if (bits & BIT(i))
1535 pe->tcam.byte[ai_idx] |= 1 << i;
1536 else
1537 pe->tcam.byte[ai_idx] &= ~(1 << i);
1538 }
1539
1540 pe->tcam.byte[MVPP2_PRS_TCAM_EN_OFFS(ai_idx)] |= enable;
1541}
1542
1543/* Get ai bits from tcam sw entry */
1544static int mvpp2_prs_tcam_ai_get(struct mvpp2_prs_entry *pe)
1545{
1546 return pe->tcam.byte[MVPP2_PRS_TCAM_AI_BYTE];
1547}
1548
1549/* Set ethertype in tcam sw entry */
1550static void mvpp2_prs_match_etype(struct mvpp2_prs_entry *pe, int offset,
1551 unsigned short ethertype)
1552{
1553 mvpp2_prs_tcam_data_byte_set(pe, offset + 0, ethertype >> 8, 0xff);
1554 mvpp2_prs_tcam_data_byte_set(pe, offset + 1, ethertype & 0xff, 0xff);
1555}
1556
1557/* Set bits in sram sw entry */
1558static void mvpp2_prs_sram_bits_set(struct mvpp2_prs_entry *pe, int bit_num,
1559 int val)
1560{
1561 pe->sram.byte[MVPP2_BIT_TO_BYTE(bit_num)] |= (val << (bit_num % 8));
1562}
1563
1564/* Clear bits in sram sw entry */
1565static void mvpp2_prs_sram_bits_clear(struct mvpp2_prs_entry *pe, int bit_num,
1566 int val)
1567{
1568 pe->sram.byte[MVPP2_BIT_TO_BYTE(bit_num)] &= ~(val << (bit_num % 8));
1569}
1570
1571/* Update ri bits in sram sw entry */
1572static void mvpp2_prs_sram_ri_update(struct mvpp2_prs_entry *pe,
1573 unsigned int bits, unsigned int mask)
1574{
1575 unsigned int i;
1576
1577 for (i = 0; i < MVPP2_PRS_SRAM_RI_CTRL_BITS; i++) {
1578 int ri_off = MVPP2_PRS_SRAM_RI_OFFS;
1579
1580 if (!(mask & BIT(i)))
1581 continue;
1582
1583 if (bits & BIT(i))
1584 mvpp2_prs_sram_bits_set(pe, ri_off + i, 1);
1585 else
1586 mvpp2_prs_sram_bits_clear(pe, ri_off + i, 1);
1587
1588 mvpp2_prs_sram_bits_set(pe, MVPP2_PRS_SRAM_RI_CTRL_OFFS + i, 1);
1589 }
1590}
1591
1592/* Obtain ri bits from sram sw entry */
1593static int mvpp2_prs_sram_ri_get(struct mvpp2_prs_entry *pe)
1594{
1595 return pe->sram.word[MVPP2_PRS_SRAM_RI_WORD];
1596}
1597
1598/* Update ai bits in sram sw entry */
1599static void mvpp2_prs_sram_ai_update(struct mvpp2_prs_entry *pe,
1600 unsigned int bits, unsigned int mask)
1601{
1602 unsigned int i;
1603 int ai_off = MVPP2_PRS_SRAM_AI_OFFS;
1604
1605 for (i = 0; i < MVPP2_PRS_SRAM_AI_CTRL_BITS; i++) {
1606
1607 if (!(mask & BIT(i)))
1608 continue;
1609
1610 if (bits & BIT(i))
1611 mvpp2_prs_sram_bits_set(pe, ai_off + i, 1);
1612 else
1613 mvpp2_prs_sram_bits_clear(pe, ai_off + i, 1);
1614
1615 mvpp2_prs_sram_bits_set(pe, MVPP2_PRS_SRAM_AI_CTRL_OFFS + i, 1);
1616 }
1617}
1618
1619/* Read ai bits from sram sw entry */
1620static int mvpp2_prs_sram_ai_get(struct mvpp2_prs_entry *pe)
1621{
1622 u8 bits;
1623 int ai_off = MVPP2_BIT_TO_BYTE(MVPP2_PRS_SRAM_AI_OFFS);
1624 int ai_en_off = ai_off + 1;
1625 int ai_shift = MVPP2_PRS_SRAM_AI_OFFS % 8;
1626
1627 bits = (pe->sram.byte[ai_off] >> ai_shift) |
1628 (pe->sram.byte[ai_en_off] << (8 - ai_shift));
1629
1630 return bits;
1631}
1632
1633/* In sram sw entry set lookup ID field of the tcam key to be used in the next
1634 * lookup interation
1635 */
1636static void mvpp2_prs_sram_next_lu_set(struct mvpp2_prs_entry *pe,
1637 unsigned int lu)
1638{
1639 int sram_next_off = MVPP2_PRS_SRAM_NEXT_LU_OFFS;
1640
1641 mvpp2_prs_sram_bits_clear(pe, sram_next_off,
1642 MVPP2_PRS_SRAM_NEXT_LU_MASK);
1643 mvpp2_prs_sram_bits_set(pe, sram_next_off, lu);
1644}
1645
1646/* In the sram sw entry set sign and value of the next lookup offset
1647 * and the offset value generated to the classifier
1648 */
1649static void mvpp2_prs_sram_shift_set(struct mvpp2_prs_entry *pe, int shift,
1650 unsigned int op)
1651{
1652 /* Set sign */
1653 if (shift < 0) {
1654 mvpp2_prs_sram_bits_set(pe, MVPP2_PRS_SRAM_SHIFT_SIGN_BIT, 1);
1655 shift = 0 - shift;
1656 } else {
1657 mvpp2_prs_sram_bits_clear(pe, MVPP2_PRS_SRAM_SHIFT_SIGN_BIT, 1);
1658 }
1659
1660 /* Set value */
1661 pe->sram.byte[MVPP2_BIT_TO_BYTE(MVPP2_PRS_SRAM_SHIFT_OFFS)] =
1662 (unsigned char)shift;
1663
1664 /* Reset and set operation */
1665 mvpp2_prs_sram_bits_clear(pe, MVPP2_PRS_SRAM_OP_SEL_SHIFT_OFFS,
1666 MVPP2_PRS_SRAM_OP_SEL_SHIFT_MASK);
1667 mvpp2_prs_sram_bits_set(pe, MVPP2_PRS_SRAM_OP_SEL_SHIFT_OFFS, op);
1668
1669 /* Set base offset as current */
1670 mvpp2_prs_sram_bits_clear(pe, MVPP2_PRS_SRAM_OP_SEL_BASE_OFFS, 1);
1671}
1672
1673/* In the sram sw entry set sign and value of the user defined offset
1674 * generated to the classifier
1675 */
1676static void mvpp2_prs_sram_offset_set(struct mvpp2_prs_entry *pe,
1677 unsigned int type, int offset,
1678 unsigned int op)
1679{
1680 /* Set sign */
1681 if (offset < 0) {
1682 mvpp2_prs_sram_bits_set(pe, MVPP2_PRS_SRAM_UDF_SIGN_BIT, 1);
1683 offset = 0 - offset;
1684 } else {
1685 mvpp2_prs_sram_bits_clear(pe, MVPP2_PRS_SRAM_UDF_SIGN_BIT, 1);
1686 }
1687
1688 /* Set value */
1689 mvpp2_prs_sram_bits_clear(pe, MVPP2_PRS_SRAM_UDF_OFFS,
1690 MVPP2_PRS_SRAM_UDF_MASK);
1691 mvpp2_prs_sram_bits_set(pe, MVPP2_PRS_SRAM_UDF_OFFS, offset);
1692 pe->sram.byte[MVPP2_BIT_TO_BYTE(MVPP2_PRS_SRAM_UDF_OFFS +
1693 MVPP2_PRS_SRAM_UDF_BITS)] &=
1694 ~(MVPP2_PRS_SRAM_UDF_MASK >> (8 - (MVPP2_PRS_SRAM_UDF_OFFS % 8)));
1695 pe->sram.byte[MVPP2_BIT_TO_BYTE(MVPP2_PRS_SRAM_UDF_OFFS +
1696 MVPP2_PRS_SRAM_UDF_BITS)] |=
1697 (offset >> (8 - (MVPP2_PRS_SRAM_UDF_OFFS % 8)));
1698
1699 /* Set offset type */
1700 mvpp2_prs_sram_bits_clear(pe, MVPP2_PRS_SRAM_UDF_TYPE_OFFS,
1701 MVPP2_PRS_SRAM_UDF_TYPE_MASK);
1702 mvpp2_prs_sram_bits_set(pe, MVPP2_PRS_SRAM_UDF_TYPE_OFFS, type);
1703
1704 /* Set offset operation */
1705 mvpp2_prs_sram_bits_clear(pe, MVPP2_PRS_SRAM_OP_SEL_UDF_OFFS,
1706 MVPP2_PRS_SRAM_OP_SEL_UDF_MASK);
1707 mvpp2_prs_sram_bits_set(pe, MVPP2_PRS_SRAM_OP_SEL_UDF_OFFS, op);
1708
1709 pe->sram.byte[MVPP2_BIT_TO_BYTE(MVPP2_PRS_SRAM_OP_SEL_UDF_OFFS +
1710 MVPP2_PRS_SRAM_OP_SEL_UDF_BITS)] &=
1711 ~(MVPP2_PRS_SRAM_OP_SEL_UDF_MASK >>
1712 (8 - (MVPP2_PRS_SRAM_OP_SEL_UDF_OFFS % 8)));
1713
1714 pe->sram.byte[MVPP2_BIT_TO_BYTE(MVPP2_PRS_SRAM_OP_SEL_UDF_OFFS +
1715 MVPP2_PRS_SRAM_OP_SEL_UDF_BITS)] |=
1716 (op >> (8 - (MVPP2_PRS_SRAM_OP_SEL_UDF_OFFS % 8)));
1717
1718 /* Set base offset as current */
1719 mvpp2_prs_sram_bits_clear(pe, MVPP2_PRS_SRAM_OP_SEL_BASE_OFFS, 1);
1720}
1721
1722/* Find parser flow entry */
1723static struct mvpp2_prs_entry *mvpp2_prs_flow_find(struct mvpp2 *priv, int flow)
1724{
1725 struct mvpp2_prs_entry *pe;
1726 int tid;
1727
1728 pe = kzalloc(sizeof(*pe), GFP_KERNEL);
1729 if (!pe)
1730 return NULL;
1731 mvpp2_prs_tcam_lu_set(pe, MVPP2_PRS_LU_FLOWS);
1732
1733 /* Go through the all entires with MVPP2_PRS_LU_FLOWS */
1734 for (tid = MVPP2_PRS_TCAM_SRAM_SIZE - 1; tid >= 0; tid--) {
1735 u8 bits;
1736
1737 if (!priv->prs_shadow[tid].valid ||
1738 priv->prs_shadow[tid].lu != MVPP2_PRS_LU_FLOWS)
1739 continue;
1740
1741 pe->index = tid;
1742 mvpp2_prs_hw_read(priv, pe);
1743 bits = mvpp2_prs_sram_ai_get(pe);
1744
1745 /* Sram store classification lookup ID in AI bits [5:0] */
1746 if ((bits & MVPP2_PRS_FLOW_ID_MASK) == flow)
1747 return pe;
1748 }
1749 kfree(pe);
1750
1751 return NULL;
1752}
1753
1754/* Return first free tcam index, seeking from start to end */
1755static int mvpp2_prs_tcam_first_free(struct mvpp2 *priv, unsigned char start,
1756 unsigned char end)
1757{
1758 int tid;
1759
1760 if (start > end)
1761 swap(start, end);
1762
1763 if (end >= MVPP2_PRS_TCAM_SRAM_SIZE)
1764 end = MVPP2_PRS_TCAM_SRAM_SIZE - 1;
1765
1766 for (tid = start; tid <= end; tid++) {
1767 if (!priv->prs_shadow[tid].valid)
1768 return tid;
1769 }
1770
1771 return -EINVAL;
1772}
1773
1774/* Enable/disable dropping all mac da's */
1775static void mvpp2_prs_mac_drop_all_set(struct mvpp2 *priv, int port, bool add)
1776{
1777 struct mvpp2_prs_entry pe;
1778
1779 if (priv->prs_shadow[MVPP2_PE_DROP_ALL].valid) {
1780 /* Entry exist - update port only */
1781 pe.index = MVPP2_PE_DROP_ALL;
1782 mvpp2_prs_hw_read(priv, &pe);
1783 } else {
1784 /* Entry doesn't exist - create new */
Markus Elfringc5b2ce22017-04-17 10:30:29 +02001785 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03001786 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_MAC);
1787 pe.index = MVPP2_PE_DROP_ALL;
1788
1789 /* Non-promiscuous mode for all ports - DROP unknown packets */
1790 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_DROP_MASK,
1791 MVPP2_PRS_RI_DROP_MASK);
1792
1793 mvpp2_prs_sram_bits_set(&pe, MVPP2_PRS_SRAM_LU_GEN_BIT, 1);
1794 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_FLOWS);
1795
1796 /* Update shadow table */
1797 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_MAC);
1798
1799 /* Mask all ports */
1800 mvpp2_prs_tcam_port_map_set(&pe, 0);
1801 }
1802
1803 /* Update port mask */
1804 mvpp2_prs_tcam_port_set(&pe, port, add);
1805
1806 mvpp2_prs_hw_write(priv, &pe);
1807}
1808
1809/* Set port to promiscuous mode */
1810static void mvpp2_prs_mac_promisc_set(struct mvpp2 *priv, int port, bool add)
1811{
1812 struct mvpp2_prs_entry pe;
1813
Joe Perchesdbedd442015-03-06 20:49:12 -08001814 /* Promiscuous mode - Accept unknown packets */
Marcin Wojtas3f518502014-07-10 16:52:13 -03001815
1816 if (priv->prs_shadow[MVPP2_PE_MAC_PROMISCUOUS].valid) {
1817 /* Entry exist - update port only */
1818 pe.index = MVPP2_PE_MAC_PROMISCUOUS;
1819 mvpp2_prs_hw_read(priv, &pe);
1820 } else {
1821 /* Entry doesn't exist - create new */
Markus Elfringc5b2ce22017-04-17 10:30:29 +02001822 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03001823 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_MAC);
1824 pe.index = MVPP2_PE_MAC_PROMISCUOUS;
1825
1826 /* Continue - set next lookup */
1827 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_DSA);
1828
1829 /* Set result info bits */
1830 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L2_UCAST,
1831 MVPP2_PRS_RI_L2_CAST_MASK);
1832
1833 /* Shift to ethertype */
1834 mvpp2_prs_sram_shift_set(&pe, 2 * ETH_ALEN,
1835 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
1836
1837 /* Mask all ports */
1838 mvpp2_prs_tcam_port_map_set(&pe, 0);
1839
1840 /* Update shadow table */
1841 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_MAC);
1842 }
1843
1844 /* Update port mask */
1845 mvpp2_prs_tcam_port_set(&pe, port, add);
1846
1847 mvpp2_prs_hw_write(priv, &pe);
1848}
1849
1850/* Accept multicast */
1851static void mvpp2_prs_mac_multi_set(struct mvpp2 *priv, int port, int index,
1852 bool add)
1853{
1854 struct mvpp2_prs_entry pe;
1855 unsigned char da_mc;
1856
1857 /* Ethernet multicast address first byte is
1858 * 0x01 for IPv4 and 0x33 for IPv6
1859 */
1860 da_mc = (index == MVPP2_PE_MAC_MC_ALL) ? 0x01 : 0x33;
1861
1862 if (priv->prs_shadow[index].valid) {
1863 /* Entry exist - update port only */
1864 pe.index = index;
1865 mvpp2_prs_hw_read(priv, &pe);
1866 } else {
1867 /* Entry doesn't exist - create new */
Markus Elfringc5b2ce22017-04-17 10:30:29 +02001868 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03001869 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_MAC);
1870 pe.index = index;
1871
1872 /* Continue - set next lookup */
1873 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_DSA);
1874
1875 /* Set result info bits */
1876 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L2_MCAST,
1877 MVPP2_PRS_RI_L2_CAST_MASK);
1878
1879 /* Update tcam entry data first byte */
1880 mvpp2_prs_tcam_data_byte_set(&pe, 0, da_mc, 0xff);
1881
1882 /* Shift to ethertype */
1883 mvpp2_prs_sram_shift_set(&pe, 2 * ETH_ALEN,
1884 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
1885
1886 /* Mask all ports */
1887 mvpp2_prs_tcam_port_map_set(&pe, 0);
1888
1889 /* Update shadow table */
1890 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_MAC);
1891 }
1892
1893 /* Update port mask */
1894 mvpp2_prs_tcam_port_set(&pe, port, add);
1895
1896 mvpp2_prs_hw_write(priv, &pe);
1897}
1898
1899/* Set entry for dsa packets */
1900static void mvpp2_prs_dsa_tag_set(struct mvpp2 *priv, int port, bool add,
1901 bool tagged, bool extend)
1902{
1903 struct mvpp2_prs_entry pe;
1904 int tid, shift;
1905
1906 if (extend) {
1907 tid = tagged ? MVPP2_PE_EDSA_TAGGED : MVPP2_PE_EDSA_UNTAGGED;
1908 shift = 8;
1909 } else {
1910 tid = tagged ? MVPP2_PE_DSA_TAGGED : MVPP2_PE_DSA_UNTAGGED;
1911 shift = 4;
1912 }
1913
1914 if (priv->prs_shadow[tid].valid) {
1915 /* Entry exist - update port only */
1916 pe.index = tid;
1917 mvpp2_prs_hw_read(priv, &pe);
1918 } else {
1919 /* Entry doesn't exist - create new */
Markus Elfringc5b2ce22017-04-17 10:30:29 +02001920 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03001921 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_DSA);
1922 pe.index = tid;
1923
1924 /* Shift 4 bytes if DSA tag or 8 bytes in case of EDSA tag*/
1925 mvpp2_prs_sram_shift_set(&pe, shift,
1926 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
1927
1928 /* Update shadow table */
1929 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_DSA);
1930
1931 if (tagged) {
1932 /* Set tagged bit in DSA tag */
1933 mvpp2_prs_tcam_data_byte_set(&pe, 0,
1934 MVPP2_PRS_TCAM_DSA_TAGGED_BIT,
1935 MVPP2_PRS_TCAM_DSA_TAGGED_BIT);
1936 /* Clear all ai bits for next iteration */
1937 mvpp2_prs_sram_ai_update(&pe, 0,
1938 MVPP2_PRS_SRAM_AI_MASK);
1939 /* If packet is tagged continue check vlans */
1940 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_VLAN);
1941 } else {
1942 /* Set result info bits to 'no vlans' */
1943 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_VLAN_NONE,
1944 MVPP2_PRS_RI_VLAN_MASK);
1945 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_L2);
1946 }
1947
1948 /* Mask all ports */
1949 mvpp2_prs_tcam_port_map_set(&pe, 0);
1950 }
1951
1952 /* Update port mask */
1953 mvpp2_prs_tcam_port_set(&pe, port, add);
1954
1955 mvpp2_prs_hw_write(priv, &pe);
1956}
1957
1958/* Set entry for dsa ethertype */
1959static void mvpp2_prs_dsa_tag_ethertype_set(struct mvpp2 *priv, int port,
1960 bool add, bool tagged, bool extend)
1961{
1962 struct mvpp2_prs_entry pe;
1963 int tid, shift, port_mask;
1964
1965 if (extend) {
1966 tid = tagged ? MVPP2_PE_ETYPE_EDSA_TAGGED :
1967 MVPP2_PE_ETYPE_EDSA_UNTAGGED;
1968 port_mask = 0;
1969 shift = 8;
1970 } else {
1971 tid = tagged ? MVPP2_PE_ETYPE_DSA_TAGGED :
1972 MVPP2_PE_ETYPE_DSA_UNTAGGED;
1973 port_mask = MVPP2_PRS_PORT_MASK;
1974 shift = 4;
1975 }
1976
1977 if (priv->prs_shadow[tid].valid) {
1978 /* Entry exist - update port only */
1979 pe.index = tid;
1980 mvpp2_prs_hw_read(priv, &pe);
1981 } else {
1982 /* Entry doesn't exist - create new */
Markus Elfringc5b2ce22017-04-17 10:30:29 +02001983 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03001984 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_DSA);
1985 pe.index = tid;
1986
1987 /* Set ethertype */
1988 mvpp2_prs_match_etype(&pe, 0, ETH_P_EDSA);
1989 mvpp2_prs_match_etype(&pe, 2, 0);
1990
1991 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_DSA_MASK,
1992 MVPP2_PRS_RI_DSA_MASK);
1993 /* Shift ethertype + 2 byte reserved + tag*/
1994 mvpp2_prs_sram_shift_set(&pe, 2 + MVPP2_ETH_TYPE_LEN + shift,
1995 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
1996
1997 /* Update shadow table */
1998 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_DSA);
1999
2000 if (tagged) {
2001 /* Set tagged bit in DSA tag */
2002 mvpp2_prs_tcam_data_byte_set(&pe,
2003 MVPP2_ETH_TYPE_LEN + 2 + 3,
2004 MVPP2_PRS_TCAM_DSA_TAGGED_BIT,
2005 MVPP2_PRS_TCAM_DSA_TAGGED_BIT);
2006 /* Clear all ai bits for next iteration */
2007 mvpp2_prs_sram_ai_update(&pe, 0,
2008 MVPP2_PRS_SRAM_AI_MASK);
2009 /* If packet is tagged continue check vlans */
2010 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_VLAN);
2011 } else {
2012 /* Set result info bits to 'no vlans' */
2013 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_VLAN_NONE,
2014 MVPP2_PRS_RI_VLAN_MASK);
2015 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_L2);
2016 }
2017 /* Mask/unmask all ports, depending on dsa type */
2018 mvpp2_prs_tcam_port_map_set(&pe, port_mask);
2019 }
2020
2021 /* Update port mask */
2022 mvpp2_prs_tcam_port_set(&pe, port, add);
2023
2024 mvpp2_prs_hw_write(priv, &pe);
2025}
2026
2027/* Search for existing single/triple vlan entry */
2028static struct mvpp2_prs_entry *mvpp2_prs_vlan_find(struct mvpp2 *priv,
2029 unsigned short tpid, int ai)
2030{
2031 struct mvpp2_prs_entry *pe;
2032 int tid;
2033
2034 pe = kzalloc(sizeof(*pe), GFP_KERNEL);
2035 if (!pe)
2036 return NULL;
2037 mvpp2_prs_tcam_lu_set(pe, MVPP2_PRS_LU_VLAN);
2038
2039 /* Go through the all entries with MVPP2_PRS_LU_VLAN */
2040 for (tid = MVPP2_PE_FIRST_FREE_TID;
2041 tid <= MVPP2_PE_LAST_FREE_TID; tid++) {
2042 unsigned int ri_bits, ai_bits;
2043 bool match;
2044
2045 if (!priv->prs_shadow[tid].valid ||
2046 priv->prs_shadow[tid].lu != MVPP2_PRS_LU_VLAN)
2047 continue;
2048
2049 pe->index = tid;
2050
2051 mvpp2_prs_hw_read(priv, pe);
2052 match = mvpp2_prs_tcam_data_cmp(pe, 0, swab16(tpid));
2053 if (!match)
2054 continue;
2055
2056 /* Get vlan type */
2057 ri_bits = mvpp2_prs_sram_ri_get(pe);
2058 ri_bits &= MVPP2_PRS_RI_VLAN_MASK;
2059
2060 /* Get current ai value from tcam */
2061 ai_bits = mvpp2_prs_tcam_ai_get(pe);
2062 /* Clear double vlan bit */
2063 ai_bits &= ~MVPP2_PRS_DBL_VLAN_AI_BIT;
2064
2065 if (ai != ai_bits)
2066 continue;
2067
2068 if (ri_bits == MVPP2_PRS_RI_VLAN_SINGLE ||
2069 ri_bits == MVPP2_PRS_RI_VLAN_TRIPLE)
2070 return pe;
2071 }
2072 kfree(pe);
2073
2074 return NULL;
2075}
2076
2077/* Add/update single/triple vlan entry */
2078static int mvpp2_prs_vlan_add(struct mvpp2 *priv, unsigned short tpid, int ai,
2079 unsigned int port_map)
2080{
2081 struct mvpp2_prs_entry *pe;
2082 int tid_aux, tid;
Sudip Mukherjee43737472014-11-01 16:59:34 +05302083 int ret = 0;
Marcin Wojtas3f518502014-07-10 16:52:13 -03002084
2085 pe = mvpp2_prs_vlan_find(priv, tpid, ai);
2086
2087 if (!pe) {
2088 /* Create new tcam entry */
2089 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_LAST_FREE_TID,
2090 MVPP2_PE_FIRST_FREE_TID);
2091 if (tid < 0)
2092 return tid;
2093
2094 pe = kzalloc(sizeof(*pe), GFP_KERNEL);
2095 if (!pe)
2096 return -ENOMEM;
2097
2098 /* Get last double vlan tid */
2099 for (tid_aux = MVPP2_PE_LAST_FREE_TID;
2100 tid_aux >= MVPP2_PE_FIRST_FREE_TID; tid_aux--) {
2101 unsigned int ri_bits;
2102
2103 if (!priv->prs_shadow[tid_aux].valid ||
2104 priv->prs_shadow[tid_aux].lu != MVPP2_PRS_LU_VLAN)
2105 continue;
2106
2107 pe->index = tid_aux;
2108 mvpp2_prs_hw_read(priv, pe);
2109 ri_bits = mvpp2_prs_sram_ri_get(pe);
2110 if ((ri_bits & MVPP2_PRS_RI_VLAN_MASK) ==
2111 MVPP2_PRS_RI_VLAN_DOUBLE)
2112 break;
2113 }
2114
Sudip Mukherjee43737472014-11-01 16:59:34 +05302115 if (tid <= tid_aux) {
2116 ret = -EINVAL;
Markus Elfringf9fd0e32017-04-17 13:50:35 +02002117 goto free_pe;
Sudip Mukherjee43737472014-11-01 16:59:34 +05302118 }
Marcin Wojtas3f518502014-07-10 16:52:13 -03002119
Markus Elfringbd6aaf52017-04-17 10:40:32 +02002120 memset(pe, 0, sizeof(*pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03002121 mvpp2_prs_tcam_lu_set(pe, MVPP2_PRS_LU_VLAN);
2122 pe->index = tid;
2123
2124 mvpp2_prs_match_etype(pe, 0, tpid);
2125
2126 mvpp2_prs_sram_next_lu_set(pe, MVPP2_PRS_LU_L2);
2127 /* Shift 4 bytes - skip 1 vlan tag */
2128 mvpp2_prs_sram_shift_set(pe, MVPP2_VLAN_TAG_LEN,
2129 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
2130 /* Clear all ai bits for next iteration */
2131 mvpp2_prs_sram_ai_update(pe, 0, MVPP2_PRS_SRAM_AI_MASK);
2132
2133 if (ai == MVPP2_PRS_SINGLE_VLAN_AI) {
2134 mvpp2_prs_sram_ri_update(pe, MVPP2_PRS_RI_VLAN_SINGLE,
2135 MVPP2_PRS_RI_VLAN_MASK);
2136 } else {
2137 ai |= MVPP2_PRS_DBL_VLAN_AI_BIT;
2138 mvpp2_prs_sram_ri_update(pe, MVPP2_PRS_RI_VLAN_TRIPLE,
2139 MVPP2_PRS_RI_VLAN_MASK);
2140 }
2141 mvpp2_prs_tcam_ai_update(pe, ai, MVPP2_PRS_SRAM_AI_MASK);
2142
2143 mvpp2_prs_shadow_set(priv, pe->index, MVPP2_PRS_LU_VLAN);
2144 }
2145 /* Update ports' mask */
2146 mvpp2_prs_tcam_port_map_set(pe, port_map);
2147
2148 mvpp2_prs_hw_write(priv, pe);
Markus Elfringf9fd0e32017-04-17 13:50:35 +02002149free_pe:
Marcin Wojtas3f518502014-07-10 16:52:13 -03002150 kfree(pe);
2151
Sudip Mukherjee43737472014-11-01 16:59:34 +05302152 return ret;
Marcin Wojtas3f518502014-07-10 16:52:13 -03002153}
2154
2155/* Get first free double vlan ai number */
2156static int mvpp2_prs_double_vlan_ai_free_get(struct mvpp2 *priv)
2157{
2158 int i;
2159
2160 for (i = 1; i < MVPP2_PRS_DBL_VLANS_MAX; i++) {
2161 if (!priv->prs_double_vlans[i])
2162 return i;
2163 }
2164
2165 return -EINVAL;
2166}
2167
2168/* Search for existing double vlan entry */
2169static struct mvpp2_prs_entry *mvpp2_prs_double_vlan_find(struct mvpp2 *priv,
2170 unsigned short tpid1,
2171 unsigned short tpid2)
2172{
2173 struct mvpp2_prs_entry *pe;
2174 int tid;
2175
2176 pe = kzalloc(sizeof(*pe), GFP_KERNEL);
2177 if (!pe)
2178 return NULL;
2179 mvpp2_prs_tcam_lu_set(pe, MVPP2_PRS_LU_VLAN);
2180
2181 /* Go through the all entries with MVPP2_PRS_LU_VLAN */
2182 for (tid = MVPP2_PE_FIRST_FREE_TID;
2183 tid <= MVPP2_PE_LAST_FREE_TID; tid++) {
2184 unsigned int ri_mask;
2185 bool match;
2186
2187 if (!priv->prs_shadow[tid].valid ||
2188 priv->prs_shadow[tid].lu != MVPP2_PRS_LU_VLAN)
2189 continue;
2190
2191 pe->index = tid;
2192 mvpp2_prs_hw_read(priv, pe);
2193
2194 match = mvpp2_prs_tcam_data_cmp(pe, 0, swab16(tpid1))
2195 && mvpp2_prs_tcam_data_cmp(pe, 4, swab16(tpid2));
2196
2197 if (!match)
2198 continue;
2199
2200 ri_mask = mvpp2_prs_sram_ri_get(pe) & MVPP2_PRS_RI_VLAN_MASK;
2201 if (ri_mask == MVPP2_PRS_RI_VLAN_DOUBLE)
2202 return pe;
2203 }
2204 kfree(pe);
2205
2206 return NULL;
2207}
2208
2209/* Add or update double vlan entry */
2210static int mvpp2_prs_double_vlan_add(struct mvpp2 *priv, unsigned short tpid1,
2211 unsigned short tpid2,
2212 unsigned int port_map)
2213{
2214 struct mvpp2_prs_entry *pe;
Sudip Mukherjee43737472014-11-01 16:59:34 +05302215 int tid_aux, tid, ai, ret = 0;
Marcin Wojtas3f518502014-07-10 16:52:13 -03002216
2217 pe = mvpp2_prs_double_vlan_find(priv, tpid1, tpid2);
2218
2219 if (!pe) {
2220 /* Create new tcam entry */
2221 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_FIRST_FREE_TID,
2222 MVPP2_PE_LAST_FREE_TID);
2223 if (tid < 0)
2224 return tid;
2225
2226 pe = kzalloc(sizeof(*pe), GFP_KERNEL);
2227 if (!pe)
2228 return -ENOMEM;
2229
2230 /* Set ai value for new double vlan entry */
2231 ai = mvpp2_prs_double_vlan_ai_free_get(priv);
Sudip Mukherjee43737472014-11-01 16:59:34 +05302232 if (ai < 0) {
2233 ret = ai;
Markus Elfringc9a7e122017-04-17 13:03:49 +02002234 goto free_pe;
Sudip Mukherjee43737472014-11-01 16:59:34 +05302235 }
Marcin Wojtas3f518502014-07-10 16:52:13 -03002236
2237 /* Get first single/triple vlan tid */
2238 for (tid_aux = MVPP2_PE_FIRST_FREE_TID;
2239 tid_aux <= MVPP2_PE_LAST_FREE_TID; tid_aux++) {
2240 unsigned int ri_bits;
2241
2242 if (!priv->prs_shadow[tid_aux].valid ||
2243 priv->prs_shadow[tid_aux].lu != MVPP2_PRS_LU_VLAN)
2244 continue;
2245
2246 pe->index = tid_aux;
2247 mvpp2_prs_hw_read(priv, pe);
2248 ri_bits = mvpp2_prs_sram_ri_get(pe);
2249 ri_bits &= MVPP2_PRS_RI_VLAN_MASK;
2250 if (ri_bits == MVPP2_PRS_RI_VLAN_SINGLE ||
2251 ri_bits == MVPP2_PRS_RI_VLAN_TRIPLE)
2252 break;
2253 }
2254
Sudip Mukherjee43737472014-11-01 16:59:34 +05302255 if (tid >= tid_aux) {
2256 ret = -ERANGE;
Markus Elfringc9a7e122017-04-17 13:03:49 +02002257 goto free_pe;
Sudip Mukherjee43737472014-11-01 16:59:34 +05302258 }
Marcin Wojtas3f518502014-07-10 16:52:13 -03002259
Markus Elfringbd6aaf52017-04-17 10:40:32 +02002260 memset(pe, 0, sizeof(*pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03002261 mvpp2_prs_tcam_lu_set(pe, MVPP2_PRS_LU_VLAN);
2262 pe->index = tid;
2263
2264 priv->prs_double_vlans[ai] = true;
2265
2266 mvpp2_prs_match_etype(pe, 0, tpid1);
2267 mvpp2_prs_match_etype(pe, 4, tpid2);
2268
2269 mvpp2_prs_sram_next_lu_set(pe, MVPP2_PRS_LU_VLAN);
2270 /* Shift 8 bytes - skip 2 vlan tags */
2271 mvpp2_prs_sram_shift_set(pe, 2 * MVPP2_VLAN_TAG_LEN,
2272 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
2273 mvpp2_prs_sram_ri_update(pe, MVPP2_PRS_RI_VLAN_DOUBLE,
2274 MVPP2_PRS_RI_VLAN_MASK);
2275 mvpp2_prs_sram_ai_update(pe, ai | MVPP2_PRS_DBL_VLAN_AI_BIT,
2276 MVPP2_PRS_SRAM_AI_MASK);
2277
2278 mvpp2_prs_shadow_set(priv, pe->index, MVPP2_PRS_LU_VLAN);
2279 }
2280
2281 /* Update ports' mask */
2282 mvpp2_prs_tcam_port_map_set(pe, port_map);
2283 mvpp2_prs_hw_write(priv, pe);
Markus Elfringc9a7e122017-04-17 13:03:49 +02002284free_pe:
Marcin Wojtas3f518502014-07-10 16:52:13 -03002285 kfree(pe);
Sudip Mukherjee43737472014-11-01 16:59:34 +05302286 return ret;
Marcin Wojtas3f518502014-07-10 16:52:13 -03002287}
2288
2289/* IPv4 header parsing for fragmentation and L4 offset */
2290static int mvpp2_prs_ip4_proto(struct mvpp2 *priv, unsigned short proto,
2291 unsigned int ri, unsigned int ri_mask)
2292{
2293 struct mvpp2_prs_entry pe;
2294 int tid;
2295
2296 if ((proto != IPPROTO_TCP) && (proto != IPPROTO_UDP) &&
2297 (proto != IPPROTO_IGMP))
2298 return -EINVAL;
2299
2300 /* Fragmented packet */
2301 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_FIRST_FREE_TID,
2302 MVPP2_PE_LAST_FREE_TID);
2303 if (tid < 0)
2304 return tid;
2305
Markus Elfringc5b2ce22017-04-17 10:30:29 +02002306 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03002307 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_IP4);
2308 pe.index = tid;
2309
2310 /* Set next lu to IPv4 */
2311 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_IP4);
2312 mvpp2_prs_sram_shift_set(&pe, 12, MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
2313 /* Set L4 offset */
2314 mvpp2_prs_sram_offset_set(&pe, MVPP2_PRS_SRAM_UDF_TYPE_L4,
2315 sizeof(struct iphdr) - 4,
2316 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD);
2317 mvpp2_prs_sram_ai_update(&pe, MVPP2_PRS_IPV4_DIP_AI_BIT,
2318 MVPP2_PRS_IPV4_DIP_AI_BIT);
2319 mvpp2_prs_sram_ri_update(&pe, ri | MVPP2_PRS_RI_IP_FRAG_MASK,
2320 ri_mask | MVPP2_PRS_RI_IP_FRAG_MASK);
2321
2322 mvpp2_prs_tcam_data_byte_set(&pe, 5, proto, MVPP2_PRS_TCAM_PROTO_MASK);
2323 mvpp2_prs_tcam_ai_update(&pe, 0, MVPP2_PRS_IPV4_DIP_AI_BIT);
2324 /* Unmask all ports */
2325 mvpp2_prs_tcam_port_map_set(&pe, MVPP2_PRS_PORT_MASK);
2326
2327 /* Update shadow table and hw entry */
2328 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_IP4);
2329 mvpp2_prs_hw_write(priv, &pe);
2330
2331 /* Not fragmented packet */
2332 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_FIRST_FREE_TID,
2333 MVPP2_PE_LAST_FREE_TID);
2334 if (tid < 0)
2335 return tid;
2336
2337 pe.index = tid;
2338 /* Clear ri before updating */
2339 pe.sram.word[MVPP2_PRS_SRAM_RI_WORD] = 0x0;
2340 pe.sram.word[MVPP2_PRS_SRAM_RI_CTRL_WORD] = 0x0;
2341 mvpp2_prs_sram_ri_update(&pe, ri, ri_mask);
2342
2343 mvpp2_prs_tcam_data_byte_set(&pe, 2, 0x00, MVPP2_PRS_TCAM_PROTO_MASK_L);
2344 mvpp2_prs_tcam_data_byte_set(&pe, 3, 0x00, MVPP2_PRS_TCAM_PROTO_MASK);
2345
2346 /* Update shadow table and hw entry */
2347 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_IP4);
2348 mvpp2_prs_hw_write(priv, &pe);
2349
2350 return 0;
2351}
2352
2353/* IPv4 L3 multicast or broadcast */
2354static int mvpp2_prs_ip4_cast(struct mvpp2 *priv, unsigned short l3_cast)
2355{
2356 struct mvpp2_prs_entry pe;
2357 int mask, tid;
2358
2359 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_FIRST_FREE_TID,
2360 MVPP2_PE_LAST_FREE_TID);
2361 if (tid < 0)
2362 return tid;
2363
Markus Elfringc5b2ce22017-04-17 10:30:29 +02002364 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03002365 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_IP4);
2366 pe.index = tid;
2367
2368 switch (l3_cast) {
2369 case MVPP2_PRS_L3_MULTI_CAST:
2370 mvpp2_prs_tcam_data_byte_set(&pe, 0, MVPP2_PRS_IPV4_MC,
2371 MVPP2_PRS_IPV4_MC_MASK);
2372 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L3_MCAST,
2373 MVPP2_PRS_RI_L3_ADDR_MASK);
2374 break;
2375 case MVPP2_PRS_L3_BROAD_CAST:
2376 mask = MVPP2_PRS_IPV4_BC_MASK;
2377 mvpp2_prs_tcam_data_byte_set(&pe, 0, mask, mask);
2378 mvpp2_prs_tcam_data_byte_set(&pe, 1, mask, mask);
2379 mvpp2_prs_tcam_data_byte_set(&pe, 2, mask, mask);
2380 mvpp2_prs_tcam_data_byte_set(&pe, 3, mask, mask);
2381 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L3_BCAST,
2382 MVPP2_PRS_RI_L3_ADDR_MASK);
2383 break;
2384 default:
2385 return -EINVAL;
2386 }
2387
2388 /* Finished: go to flowid generation */
2389 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_FLOWS);
2390 mvpp2_prs_sram_bits_set(&pe, MVPP2_PRS_SRAM_LU_GEN_BIT, 1);
2391
2392 mvpp2_prs_tcam_ai_update(&pe, MVPP2_PRS_IPV4_DIP_AI_BIT,
2393 MVPP2_PRS_IPV4_DIP_AI_BIT);
2394 /* Unmask all ports */
2395 mvpp2_prs_tcam_port_map_set(&pe, MVPP2_PRS_PORT_MASK);
2396
2397 /* Update shadow table and hw entry */
2398 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_IP4);
2399 mvpp2_prs_hw_write(priv, &pe);
2400
2401 return 0;
2402}
2403
2404/* Set entries for protocols over IPv6 */
2405static int mvpp2_prs_ip6_proto(struct mvpp2 *priv, unsigned short proto,
2406 unsigned int ri, unsigned int ri_mask)
2407{
2408 struct mvpp2_prs_entry pe;
2409 int tid;
2410
2411 if ((proto != IPPROTO_TCP) && (proto != IPPROTO_UDP) &&
2412 (proto != IPPROTO_ICMPV6) && (proto != IPPROTO_IPIP))
2413 return -EINVAL;
2414
2415 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_FIRST_FREE_TID,
2416 MVPP2_PE_LAST_FREE_TID);
2417 if (tid < 0)
2418 return tid;
2419
Markus Elfringc5b2ce22017-04-17 10:30:29 +02002420 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03002421 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_IP6);
2422 pe.index = tid;
2423
2424 /* Finished: go to flowid generation */
2425 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_FLOWS);
2426 mvpp2_prs_sram_bits_set(&pe, MVPP2_PRS_SRAM_LU_GEN_BIT, 1);
2427 mvpp2_prs_sram_ri_update(&pe, ri, ri_mask);
2428 mvpp2_prs_sram_offset_set(&pe, MVPP2_PRS_SRAM_UDF_TYPE_L4,
2429 sizeof(struct ipv6hdr) - 6,
2430 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD);
2431
2432 mvpp2_prs_tcam_data_byte_set(&pe, 0, proto, MVPP2_PRS_TCAM_PROTO_MASK);
2433 mvpp2_prs_tcam_ai_update(&pe, MVPP2_PRS_IPV6_NO_EXT_AI_BIT,
2434 MVPP2_PRS_IPV6_NO_EXT_AI_BIT);
2435 /* Unmask all ports */
2436 mvpp2_prs_tcam_port_map_set(&pe, MVPP2_PRS_PORT_MASK);
2437
2438 /* Write HW */
2439 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_IP6);
2440 mvpp2_prs_hw_write(priv, &pe);
2441
2442 return 0;
2443}
2444
2445/* IPv6 L3 multicast entry */
2446static int mvpp2_prs_ip6_cast(struct mvpp2 *priv, unsigned short l3_cast)
2447{
2448 struct mvpp2_prs_entry pe;
2449 int tid;
2450
2451 if (l3_cast != MVPP2_PRS_L3_MULTI_CAST)
2452 return -EINVAL;
2453
2454 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_FIRST_FREE_TID,
2455 MVPP2_PE_LAST_FREE_TID);
2456 if (tid < 0)
2457 return tid;
2458
Markus Elfringc5b2ce22017-04-17 10:30:29 +02002459 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03002460 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_IP6);
2461 pe.index = tid;
2462
2463 /* Finished: go to flowid generation */
2464 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_IP6);
2465 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L3_MCAST,
2466 MVPP2_PRS_RI_L3_ADDR_MASK);
2467 mvpp2_prs_sram_ai_update(&pe, MVPP2_PRS_IPV6_NO_EXT_AI_BIT,
2468 MVPP2_PRS_IPV6_NO_EXT_AI_BIT);
2469 /* Shift back to IPv6 NH */
2470 mvpp2_prs_sram_shift_set(&pe, -18, MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
2471
2472 mvpp2_prs_tcam_data_byte_set(&pe, 0, MVPP2_PRS_IPV6_MC,
2473 MVPP2_PRS_IPV6_MC_MASK);
2474 mvpp2_prs_tcam_ai_update(&pe, 0, MVPP2_PRS_IPV6_NO_EXT_AI_BIT);
2475 /* Unmask all ports */
2476 mvpp2_prs_tcam_port_map_set(&pe, MVPP2_PRS_PORT_MASK);
2477
2478 /* Update shadow table and hw entry */
2479 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_IP6);
2480 mvpp2_prs_hw_write(priv, &pe);
2481
2482 return 0;
2483}
2484
2485/* Parser per-port initialization */
2486static void mvpp2_prs_hw_port_init(struct mvpp2 *priv, int port, int lu_first,
2487 int lu_max, int offset)
2488{
2489 u32 val;
2490
2491 /* Set lookup ID */
2492 val = mvpp2_read(priv, MVPP2_PRS_INIT_LOOKUP_REG);
2493 val &= ~MVPP2_PRS_PORT_LU_MASK(port);
2494 val |= MVPP2_PRS_PORT_LU_VAL(port, lu_first);
2495 mvpp2_write(priv, MVPP2_PRS_INIT_LOOKUP_REG, val);
2496
2497 /* Set maximum number of loops for packet received from port */
2498 val = mvpp2_read(priv, MVPP2_PRS_MAX_LOOP_REG(port));
2499 val &= ~MVPP2_PRS_MAX_LOOP_MASK(port);
2500 val |= MVPP2_PRS_MAX_LOOP_VAL(port, lu_max);
2501 mvpp2_write(priv, MVPP2_PRS_MAX_LOOP_REG(port), val);
2502
2503 /* Set initial offset for packet header extraction for the first
2504 * searching loop
2505 */
2506 val = mvpp2_read(priv, MVPP2_PRS_INIT_OFFS_REG(port));
2507 val &= ~MVPP2_PRS_INIT_OFF_MASK(port);
2508 val |= MVPP2_PRS_INIT_OFF_VAL(port, offset);
2509 mvpp2_write(priv, MVPP2_PRS_INIT_OFFS_REG(port), val);
2510}
2511
2512/* Default flow entries initialization for all ports */
2513static void mvpp2_prs_def_flow_init(struct mvpp2 *priv)
2514{
2515 struct mvpp2_prs_entry pe;
2516 int port;
2517
2518 for (port = 0; port < MVPP2_MAX_PORTS; port++) {
Markus Elfringc5b2ce22017-04-17 10:30:29 +02002519 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03002520 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_FLOWS);
2521 pe.index = MVPP2_PE_FIRST_DEFAULT_FLOW - port;
2522
2523 /* Mask all ports */
2524 mvpp2_prs_tcam_port_map_set(&pe, 0);
2525
2526 /* Set flow ID*/
2527 mvpp2_prs_sram_ai_update(&pe, port, MVPP2_PRS_FLOW_ID_MASK);
2528 mvpp2_prs_sram_bits_set(&pe, MVPP2_PRS_SRAM_LU_DONE_BIT, 1);
2529
2530 /* Update shadow table and hw entry */
2531 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_FLOWS);
2532 mvpp2_prs_hw_write(priv, &pe);
2533 }
2534}
2535
2536/* Set default entry for Marvell Header field */
2537static void mvpp2_prs_mh_init(struct mvpp2 *priv)
2538{
2539 struct mvpp2_prs_entry pe;
2540
Markus Elfringc5b2ce22017-04-17 10:30:29 +02002541 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03002542
2543 pe.index = MVPP2_PE_MH_DEFAULT;
2544 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_MH);
2545 mvpp2_prs_sram_shift_set(&pe, MVPP2_MH_SIZE,
2546 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
2547 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_MAC);
2548
2549 /* Unmask all ports */
2550 mvpp2_prs_tcam_port_map_set(&pe, MVPP2_PRS_PORT_MASK);
2551
2552 /* Update shadow table and hw entry */
2553 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_MH);
2554 mvpp2_prs_hw_write(priv, &pe);
2555}
2556
2557/* Set default entires (place holder) for promiscuous, non-promiscuous and
2558 * multicast MAC addresses
2559 */
2560static void mvpp2_prs_mac_init(struct mvpp2 *priv)
2561{
2562 struct mvpp2_prs_entry pe;
2563
Markus Elfringc5b2ce22017-04-17 10:30:29 +02002564 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03002565
2566 /* Non-promiscuous mode for all ports - DROP unknown packets */
2567 pe.index = MVPP2_PE_MAC_NON_PROMISCUOUS;
2568 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_MAC);
2569
2570 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_DROP_MASK,
2571 MVPP2_PRS_RI_DROP_MASK);
2572 mvpp2_prs_sram_bits_set(&pe, MVPP2_PRS_SRAM_LU_GEN_BIT, 1);
2573 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_FLOWS);
2574
2575 /* Unmask all ports */
2576 mvpp2_prs_tcam_port_map_set(&pe, MVPP2_PRS_PORT_MASK);
2577
2578 /* Update shadow table and hw entry */
2579 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_MAC);
2580 mvpp2_prs_hw_write(priv, &pe);
2581
2582 /* place holders only - no ports */
2583 mvpp2_prs_mac_drop_all_set(priv, 0, false);
2584 mvpp2_prs_mac_promisc_set(priv, 0, false);
2585 mvpp2_prs_mac_multi_set(priv, MVPP2_PE_MAC_MC_ALL, 0, false);
2586 mvpp2_prs_mac_multi_set(priv, MVPP2_PE_MAC_MC_IP6, 0, false);
2587}
2588
2589/* Set default entries for various types of dsa packets */
2590static void mvpp2_prs_dsa_init(struct mvpp2 *priv)
2591{
2592 struct mvpp2_prs_entry pe;
2593
2594 /* None tagged EDSA entry - place holder */
2595 mvpp2_prs_dsa_tag_set(priv, 0, false, MVPP2_PRS_UNTAGGED,
2596 MVPP2_PRS_EDSA);
2597
2598 /* Tagged EDSA entry - place holder */
2599 mvpp2_prs_dsa_tag_set(priv, 0, false, MVPP2_PRS_TAGGED, MVPP2_PRS_EDSA);
2600
2601 /* None tagged DSA entry - place holder */
2602 mvpp2_prs_dsa_tag_set(priv, 0, false, MVPP2_PRS_UNTAGGED,
2603 MVPP2_PRS_DSA);
2604
2605 /* Tagged DSA entry - place holder */
2606 mvpp2_prs_dsa_tag_set(priv, 0, false, MVPP2_PRS_TAGGED, MVPP2_PRS_DSA);
2607
2608 /* None tagged EDSA ethertype entry - place holder*/
2609 mvpp2_prs_dsa_tag_ethertype_set(priv, 0, false,
2610 MVPP2_PRS_UNTAGGED, MVPP2_PRS_EDSA);
2611
2612 /* Tagged EDSA ethertype entry - place holder*/
2613 mvpp2_prs_dsa_tag_ethertype_set(priv, 0, false,
2614 MVPP2_PRS_TAGGED, MVPP2_PRS_EDSA);
2615
2616 /* None tagged DSA ethertype entry */
2617 mvpp2_prs_dsa_tag_ethertype_set(priv, 0, true,
2618 MVPP2_PRS_UNTAGGED, MVPP2_PRS_DSA);
2619
2620 /* Tagged DSA ethertype entry */
2621 mvpp2_prs_dsa_tag_ethertype_set(priv, 0, true,
2622 MVPP2_PRS_TAGGED, MVPP2_PRS_DSA);
2623
2624 /* Set default entry, in case DSA or EDSA tag not found */
Markus Elfringc5b2ce22017-04-17 10:30:29 +02002625 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03002626 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_DSA);
2627 pe.index = MVPP2_PE_DSA_DEFAULT;
2628 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_VLAN);
2629
2630 /* Shift 0 bytes */
2631 mvpp2_prs_sram_shift_set(&pe, 0, MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
2632 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_MAC);
2633
2634 /* Clear all sram ai bits for next iteration */
2635 mvpp2_prs_sram_ai_update(&pe, 0, MVPP2_PRS_SRAM_AI_MASK);
2636
2637 /* Unmask all ports */
2638 mvpp2_prs_tcam_port_map_set(&pe, MVPP2_PRS_PORT_MASK);
2639
2640 mvpp2_prs_hw_write(priv, &pe);
2641}
2642
2643/* Match basic ethertypes */
2644static int mvpp2_prs_etype_init(struct mvpp2 *priv)
2645{
2646 struct mvpp2_prs_entry pe;
2647 int tid;
2648
2649 /* Ethertype: PPPoE */
2650 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_FIRST_FREE_TID,
2651 MVPP2_PE_LAST_FREE_TID);
2652 if (tid < 0)
2653 return tid;
2654
Markus Elfringc5b2ce22017-04-17 10:30:29 +02002655 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03002656 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_L2);
2657 pe.index = tid;
2658
2659 mvpp2_prs_match_etype(&pe, 0, ETH_P_PPP_SES);
2660
2661 mvpp2_prs_sram_shift_set(&pe, MVPP2_PPPOE_HDR_SIZE,
2662 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
2663 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_PPPOE);
2664 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_PPPOE_MASK,
2665 MVPP2_PRS_RI_PPPOE_MASK);
2666
2667 /* Update shadow table and hw entry */
2668 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_L2);
2669 priv->prs_shadow[pe.index].udf = MVPP2_PRS_UDF_L2_DEF;
2670 priv->prs_shadow[pe.index].finish = false;
2671 mvpp2_prs_shadow_ri_set(priv, pe.index, MVPP2_PRS_RI_PPPOE_MASK,
2672 MVPP2_PRS_RI_PPPOE_MASK);
2673 mvpp2_prs_hw_write(priv, &pe);
2674
2675 /* Ethertype: ARP */
2676 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_FIRST_FREE_TID,
2677 MVPP2_PE_LAST_FREE_TID);
2678 if (tid < 0)
2679 return tid;
2680
Markus Elfringc5b2ce22017-04-17 10:30:29 +02002681 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03002682 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_L2);
2683 pe.index = tid;
2684
2685 mvpp2_prs_match_etype(&pe, 0, ETH_P_ARP);
2686
2687 /* Generate flow in the next iteration*/
2688 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_FLOWS);
2689 mvpp2_prs_sram_bits_set(&pe, MVPP2_PRS_SRAM_LU_GEN_BIT, 1);
2690 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L3_ARP,
2691 MVPP2_PRS_RI_L3_PROTO_MASK);
2692 /* Set L3 offset */
2693 mvpp2_prs_sram_offset_set(&pe, MVPP2_PRS_SRAM_UDF_TYPE_L3,
2694 MVPP2_ETH_TYPE_LEN,
2695 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD);
2696
2697 /* Update shadow table and hw entry */
2698 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_L2);
2699 priv->prs_shadow[pe.index].udf = MVPP2_PRS_UDF_L2_DEF;
2700 priv->prs_shadow[pe.index].finish = true;
2701 mvpp2_prs_shadow_ri_set(priv, pe.index, MVPP2_PRS_RI_L3_ARP,
2702 MVPP2_PRS_RI_L3_PROTO_MASK);
2703 mvpp2_prs_hw_write(priv, &pe);
2704
2705 /* Ethertype: LBTD */
2706 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_FIRST_FREE_TID,
2707 MVPP2_PE_LAST_FREE_TID);
2708 if (tid < 0)
2709 return tid;
2710
Markus Elfringc5b2ce22017-04-17 10:30:29 +02002711 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03002712 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_L2);
2713 pe.index = tid;
2714
2715 mvpp2_prs_match_etype(&pe, 0, MVPP2_IP_LBDT_TYPE);
2716
2717 /* Generate flow in the next iteration*/
2718 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_FLOWS);
2719 mvpp2_prs_sram_bits_set(&pe, MVPP2_PRS_SRAM_LU_GEN_BIT, 1);
2720 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_CPU_CODE_RX_SPEC |
2721 MVPP2_PRS_RI_UDF3_RX_SPECIAL,
2722 MVPP2_PRS_RI_CPU_CODE_MASK |
2723 MVPP2_PRS_RI_UDF3_MASK);
2724 /* Set L3 offset */
2725 mvpp2_prs_sram_offset_set(&pe, MVPP2_PRS_SRAM_UDF_TYPE_L3,
2726 MVPP2_ETH_TYPE_LEN,
2727 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD);
2728
2729 /* Update shadow table and hw entry */
2730 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_L2);
2731 priv->prs_shadow[pe.index].udf = MVPP2_PRS_UDF_L2_DEF;
2732 priv->prs_shadow[pe.index].finish = true;
2733 mvpp2_prs_shadow_ri_set(priv, pe.index, MVPP2_PRS_RI_CPU_CODE_RX_SPEC |
2734 MVPP2_PRS_RI_UDF3_RX_SPECIAL,
2735 MVPP2_PRS_RI_CPU_CODE_MASK |
2736 MVPP2_PRS_RI_UDF3_MASK);
2737 mvpp2_prs_hw_write(priv, &pe);
2738
2739 /* Ethertype: IPv4 without options */
2740 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_FIRST_FREE_TID,
2741 MVPP2_PE_LAST_FREE_TID);
2742 if (tid < 0)
2743 return tid;
2744
Markus Elfringc5b2ce22017-04-17 10:30:29 +02002745 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03002746 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_L2);
2747 pe.index = tid;
2748
2749 mvpp2_prs_match_etype(&pe, 0, ETH_P_IP);
2750 mvpp2_prs_tcam_data_byte_set(&pe, MVPP2_ETH_TYPE_LEN,
2751 MVPP2_PRS_IPV4_HEAD | MVPP2_PRS_IPV4_IHL,
2752 MVPP2_PRS_IPV4_HEAD_MASK |
2753 MVPP2_PRS_IPV4_IHL_MASK);
2754
2755 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_IP4);
2756 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L3_IP4,
2757 MVPP2_PRS_RI_L3_PROTO_MASK);
2758 /* Skip eth_type + 4 bytes of IP header */
2759 mvpp2_prs_sram_shift_set(&pe, MVPP2_ETH_TYPE_LEN + 4,
2760 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
2761 /* Set L3 offset */
2762 mvpp2_prs_sram_offset_set(&pe, MVPP2_PRS_SRAM_UDF_TYPE_L3,
2763 MVPP2_ETH_TYPE_LEN,
2764 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD);
2765
2766 /* Update shadow table and hw entry */
2767 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_L2);
2768 priv->prs_shadow[pe.index].udf = MVPP2_PRS_UDF_L2_DEF;
2769 priv->prs_shadow[pe.index].finish = false;
2770 mvpp2_prs_shadow_ri_set(priv, pe.index, MVPP2_PRS_RI_L3_IP4,
2771 MVPP2_PRS_RI_L3_PROTO_MASK);
2772 mvpp2_prs_hw_write(priv, &pe);
2773
2774 /* Ethertype: IPv4 with options */
2775 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_FIRST_FREE_TID,
2776 MVPP2_PE_LAST_FREE_TID);
2777 if (tid < 0)
2778 return tid;
2779
2780 pe.index = tid;
2781
2782 /* Clear tcam data before updating */
2783 pe.tcam.byte[MVPP2_PRS_TCAM_DATA_BYTE(MVPP2_ETH_TYPE_LEN)] = 0x0;
2784 pe.tcam.byte[MVPP2_PRS_TCAM_DATA_BYTE_EN(MVPP2_ETH_TYPE_LEN)] = 0x0;
2785
2786 mvpp2_prs_tcam_data_byte_set(&pe, MVPP2_ETH_TYPE_LEN,
2787 MVPP2_PRS_IPV4_HEAD,
2788 MVPP2_PRS_IPV4_HEAD_MASK);
2789
2790 /* Clear ri before updating */
2791 pe.sram.word[MVPP2_PRS_SRAM_RI_WORD] = 0x0;
2792 pe.sram.word[MVPP2_PRS_SRAM_RI_CTRL_WORD] = 0x0;
2793 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L3_IP4_OPT,
2794 MVPP2_PRS_RI_L3_PROTO_MASK);
2795
2796 /* Update shadow table and hw entry */
2797 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_L2);
2798 priv->prs_shadow[pe.index].udf = MVPP2_PRS_UDF_L2_DEF;
2799 priv->prs_shadow[pe.index].finish = false;
2800 mvpp2_prs_shadow_ri_set(priv, pe.index, MVPP2_PRS_RI_L3_IP4_OPT,
2801 MVPP2_PRS_RI_L3_PROTO_MASK);
2802 mvpp2_prs_hw_write(priv, &pe);
2803
2804 /* Ethertype: IPv6 without options */
2805 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_FIRST_FREE_TID,
2806 MVPP2_PE_LAST_FREE_TID);
2807 if (tid < 0)
2808 return tid;
2809
Markus Elfringc5b2ce22017-04-17 10:30:29 +02002810 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03002811 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_L2);
2812 pe.index = tid;
2813
2814 mvpp2_prs_match_etype(&pe, 0, ETH_P_IPV6);
2815
2816 /* Skip DIP of IPV6 header */
2817 mvpp2_prs_sram_shift_set(&pe, MVPP2_ETH_TYPE_LEN + 8 +
2818 MVPP2_MAX_L3_ADDR_SIZE,
2819 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
2820 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_IP6);
2821 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L3_IP6,
2822 MVPP2_PRS_RI_L3_PROTO_MASK);
2823 /* Set L3 offset */
2824 mvpp2_prs_sram_offset_set(&pe, MVPP2_PRS_SRAM_UDF_TYPE_L3,
2825 MVPP2_ETH_TYPE_LEN,
2826 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD);
2827
2828 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_L2);
2829 priv->prs_shadow[pe.index].udf = MVPP2_PRS_UDF_L2_DEF;
2830 priv->prs_shadow[pe.index].finish = false;
2831 mvpp2_prs_shadow_ri_set(priv, pe.index, MVPP2_PRS_RI_L3_IP6,
2832 MVPP2_PRS_RI_L3_PROTO_MASK);
2833 mvpp2_prs_hw_write(priv, &pe);
2834
2835 /* Default entry for MVPP2_PRS_LU_L2 - Unknown ethtype */
2836 memset(&pe, 0, sizeof(struct mvpp2_prs_entry));
2837 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_L2);
2838 pe.index = MVPP2_PE_ETH_TYPE_UN;
2839
2840 /* Unmask all ports */
2841 mvpp2_prs_tcam_port_map_set(&pe, MVPP2_PRS_PORT_MASK);
2842
2843 /* Generate flow in the next iteration*/
2844 mvpp2_prs_sram_bits_set(&pe, MVPP2_PRS_SRAM_LU_GEN_BIT, 1);
2845 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_FLOWS);
2846 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L3_UN,
2847 MVPP2_PRS_RI_L3_PROTO_MASK);
2848 /* Set L3 offset even it's unknown L3 */
2849 mvpp2_prs_sram_offset_set(&pe, MVPP2_PRS_SRAM_UDF_TYPE_L3,
2850 MVPP2_ETH_TYPE_LEN,
2851 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD);
2852
2853 /* Update shadow table and hw entry */
2854 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_L2);
2855 priv->prs_shadow[pe.index].udf = MVPP2_PRS_UDF_L2_DEF;
2856 priv->prs_shadow[pe.index].finish = true;
2857 mvpp2_prs_shadow_ri_set(priv, pe.index, MVPP2_PRS_RI_L3_UN,
2858 MVPP2_PRS_RI_L3_PROTO_MASK);
2859 mvpp2_prs_hw_write(priv, &pe);
2860
2861 return 0;
2862}
2863
2864/* Configure vlan entries and detect up to 2 successive VLAN tags.
2865 * Possible options:
2866 * 0x8100, 0x88A8
2867 * 0x8100, 0x8100
2868 * 0x8100
2869 * 0x88A8
2870 */
2871static int mvpp2_prs_vlan_init(struct platform_device *pdev, struct mvpp2 *priv)
2872{
2873 struct mvpp2_prs_entry pe;
2874 int err;
2875
2876 priv->prs_double_vlans = devm_kcalloc(&pdev->dev, sizeof(bool),
2877 MVPP2_PRS_DBL_VLANS_MAX,
2878 GFP_KERNEL);
2879 if (!priv->prs_double_vlans)
2880 return -ENOMEM;
2881
2882 /* Double VLAN: 0x8100, 0x88A8 */
2883 err = mvpp2_prs_double_vlan_add(priv, ETH_P_8021Q, ETH_P_8021AD,
2884 MVPP2_PRS_PORT_MASK);
2885 if (err)
2886 return err;
2887
2888 /* Double VLAN: 0x8100, 0x8100 */
2889 err = mvpp2_prs_double_vlan_add(priv, ETH_P_8021Q, ETH_P_8021Q,
2890 MVPP2_PRS_PORT_MASK);
2891 if (err)
2892 return err;
2893
2894 /* Single VLAN: 0x88a8 */
2895 err = mvpp2_prs_vlan_add(priv, ETH_P_8021AD, MVPP2_PRS_SINGLE_VLAN_AI,
2896 MVPP2_PRS_PORT_MASK);
2897 if (err)
2898 return err;
2899
2900 /* Single VLAN: 0x8100 */
2901 err = mvpp2_prs_vlan_add(priv, ETH_P_8021Q, MVPP2_PRS_SINGLE_VLAN_AI,
2902 MVPP2_PRS_PORT_MASK);
2903 if (err)
2904 return err;
2905
2906 /* Set default double vlan entry */
Markus Elfringc5b2ce22017-04-17 10:30:29 +02002907 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03002908 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_VLAN);
2909 pe.index = MVPP2_PE_VLAN_DBL;
2910
2911 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_L2);
2912 /* Clear ai for next iterations */
2913 mvpp2_prs_sram_ai_update(&pe, 0, MVPP2_PRS_SRAM_AI_MASK);
2914 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_VLAN_DOUBLE,
2915 MVPP2_PRS_RI_VLAN_MASK);
2916
2917 mvpp2_prs_tcam_ai_update(&pe, MVPP2_PRS_DBL_VLAN_AI_BIT,
2918 MVPP2_PRS_DBL_VLAN_AI_BIT);
2919 /* Unmask all ports */
2920 mvpp2_prs_tcam_port_map_set(&pe, MVPP2_PRS_PORT_MASK);
2921
2922 /* Update shadow table and hw entry */
2923 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_VLAN);
2924 mvpp2_prs_hw_write(priv, &pe);
2925
2926 /* Set default vlan none entry */
Markus Elfringc5b2ce22017-04-17 10:30:29 +02002927 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03002928 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_VLAN);
2929 pe.index = MVPP2_PE_VLAN_NONE;
2930
2931 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_L2);
2932 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_VLAN_NONE,
2933 MVPP2_PRS_RI_VLAN_MASK);
2934
2935 /* Unmask all ports */
2936 mvpp2_prs_tcam_port_map_set(&pe, MVPP2_PRS_PORT_MASK);
2937
2938 /* Update shadow table and hw entry */
2939 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_VLAN);
2940 mvpp2_prs_hw_write(priv, &pe);
2941
2942 return 0;
2943}
2944
2945/* Set entries for PPPoE ethertype */
2946static int mvpp2_prs_pppoe_init(struct mvpp2 *priv)
2947{
2948 struct mvpp2_prs_entry pe;
2949 int tid;
2950
2951 /* IPv4 over PPPoE with options */
2952 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_FIRST_FREE_TID,
2953 MVPP2_PE_LAST_FREE_TID);
2954 if (tid < 0)
2955 return tid;
2956
Markus Elfringc5b2ce22017-04-17 10:30:29 +02002957 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03002958 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_PPPOE);
2959 pe.index = tid;
2960
2961 mvpp2_prs_match_etype(&pe, 0, PPP_IP);
2962
2963 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_IP4);
2964 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L3_IP4_OPT,
2965 MVPP2_PRS_RI_L3_PROTO_MASK);
2966 /* Skip eth_type + 4 bytes of IP header */
2967 mvpp2_prs_sram_shift_set(&pe, MVPP2_ETH_TYPE_LEN + 4,
2968 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
2969 /* Set L3 offset */
2970 mvpp2_prs_sram_offset_set(&pe, MVPP2_PRS_SRAM_UDF_TYPE_L3,
2971 MVPP2_ETH_TYPE_LEN,
2972 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD);
2973
2974 /* Update shadow table and hw entry */
2975 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_PPPOE);
2976 mvpp2_prs_hw_write(priv, &pe);
2977
2978 /* IPv4 over PPPoE without options */
2979 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_FIRST_FREE_TID,
2980 MVPP2_PE_LAST_FREE_TID);
2981 if (tid < 0)
2982 return tid;
2983
2984 pe.index = tid;
2985
2986 mvpp2_prs_tcam_data_byte_set(&pe, MVPP2_ETH_TYPE_LEN,
2987 MVPP2_PRS_IPV4_HEAD | MVPP2_PRS_IPV4_IHL,
2988 MVPP2_PRS_IPV4_HEAD_MASK |
2989 MVPP2_PRS_IPV4_IHL_MASK);
2990
2991 /* Clear ri before updating */
2992 pe.sram.word[MVPP2_PRS_SRAM_RI_WORD] = 0x0;
2993 pe.sram.word[MVPP2_PRS_SRAM_RI_CTRL_WORD] = 0x0;
2994 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L3_IP4,
2995 MVPP2_PRS_RI_L3_PROTO_MASK);
2996
2997 /* Update shadow table and hw entry */
2998 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_PPPOE);
2999 mvpp2_prs_hw_write(priv, &pe);
3000
3001 /* IPv6 over PPPoE */
3002 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_FIRST_FREE_TID,
3003 MVPP2_PE_LAST_FREE_TID);
3004 if (tid < 0)
3005 return tid;
3006
Markus Elfringc5b2ce22017-04-17 10:30:29 +02003007 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03003008 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_PPPOE);
3009 pe.index = tid;
3010
3011 mvpp2_prs_match_etype(&pe, 0, PPP_IPV6);
3012
3013 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_IP6);
3014 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L3_IP6,
3015 MVPP2_PRS_RI_L3_PROTO_MASK);
3016 /* Skip eth_type + 4 bytes of IPv6 header */
3017 mvpp2_prs_sram_shift_set(&pe, MVPP2_ETH_TYPE_LEN + 4,
3018 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
3019 /* Set L3 offset */
3020 mvpp2_prs_sram_offset_set(&pe, MVPP2_PRS_SRAM_UDF_TYPE_L3,
3021 MVPP2_ETH_TYPE_LEN,
3022 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD);
3023
3024 /* Update shadow table and hw entry */
3025 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_PPPOE);
3026 mvpp2_prs_hw_write(priv, &pe);
3027
3028 /* Non-IP over PPPoE */
3029 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_FIRST_FREE_TID,
3030 MVPP2_PE_LAST_FREE_TID);
3031 if (tid < 0)
3032 return tid;
3033
Markus Elfringc5b2ce22017-04-17 10:30:29 +02003034 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03003035 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_PPPOE);
3036 pe.index = tid;
3037
3038 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L3_UN,
3039 MVPP2_PRS_RI_L3_PROTO_MASK);
3040
3041 /* Finished: go to flowid generation */
3042 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_FLOWS);
3043 mvpp2_prs_sram_bits_set(&pe, MVPP2_PRS_SRAM_LU_GEN_BIT, 1);
3044 /* Set L3 offset even if it's unknown L3 */
3045 mvpp2_prs_sram_offset_set(&pe, MVPP2_PRS_SRAM_UDF_TYPE_L3,
3046 MVPP2_ETH_TYPE_LEN,
3047 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD);
3048
3049 /* Update shadow table and hw entry */
3050 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_PPPOE);
3051 mvpp2_prs_hw_write(priv, &pe);
3052
3053 return 0;
3054}
3055
3056/* Initialize entries for IPv4 */
3057static int mvpp2_prs_ip4_init(struct mvpp2 *priv)
3058{
3059 struct mvpp2_prs_entry pe;
3060 int err;
3061
3062 /* Set entries for TCP, UDP and IGMP over IPv4 */
3063 err = mvpp2_prs_ip4_proto(priv, IPPROTO_TCP, MVPP2_PRS_RI_L4_TCP,
3064 MVPP2_PRS_RI_L4_PROTO_MASK);
3065 if (err)
3066 return err;
3067
3068 err = mvpp2_prs_ip4_proto(priv, IPPROTO_UDP, MVPP2_PRS_RI_L4_UDP,
3069 MVPP2_PRS_RI_L4_PROTO_MASK);
3070 if (err)
3071 return err;
3072
3073 err = mvpp2_prs_ip4_proto(priv, IPPROTO_IGMP,
3074 MVPP2_PRS_RI_CPU_CODE_RX_SPEC |
3075 MVPP2_PRS_RI_UDF3_RX_SPECIAL,
3076 MVPP2_PRS_RI_CPU_CODE_MASK |
3077 MVPP2_PRS_RI_UDF3_MASK);
3078 if (err)
3079 return err;
3080
3081 /* IPv4 Broadcast */
3082 err = mvpp2_prs_ip4_cast(priv, MVPP2_PRS_L3_BROAD_CAST);
3083 if (err)
3084 return err;
3085
3086 /* IPv4 Multicast */
3087 err = mvpp2_prs_ip4_cast(priv, MVPP2_PRS_L3_MULTI_CAST);
3088 if (err)
3089 return err;
3090
3091 /* Default IPv4 entry for unknown protocols */
Markus Elfringc5b2ce22017-04-17 10:30:29 +02003092 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03003093 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_IP4);
3094 pe.index = MVPP2_PE_IP4_PROTO_UN;
3095
3096 /* Set next lu to IPv4 */
3097 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_IP4);
3098 mvpp2_prs_sram_shift_set(&pe, 12, MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
3099 /* Set L4 offset */
3100 mvpp2_prs_sram_offset_set(&pe, MVPP2_PRS_SRAM_UDF_TYPE_L4,
3101 sizeof(struct iphdr) - 4,
3102 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD);
3103 mvpp2_prs_sram_ai_update(&pe, MVPP2_PRS_IPV4_DIP_AI_BIT,
3104 MVPP2_PRS_IPV4_DIP_AI_BIT);
3105 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L4_OTHER,
3106 MVPP2_PRS_RI_L4_PROTO_MASK);
3107
3108 mvpp2_prs_tcam_ai_update(&pe, 0, MVPP2_PRS_IPV4_DIP_AI_BIT);
3109 /* Unmask all ports */
3110 mvpp2_prs_tcam_port_map_set(&pe, MVPP2_PRS_PORT_MASK);
3111
3112 /* Update shadow table and hw entry */
3113 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_IP4);
3114 mvpp2_prs_hw_write(priv, &pe);
3115
3116 /* Default IPv4 entry for unicast address */
Markus Elfringc5b2ce22017-04-17 10:30:29 +02003117 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03003118 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_IP4);
3119 pe.index = MVPP2_PE_IP4_ADDR_UN;
3120
3121 /* Finished: go to flowid generation */
3122 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_FLOWS);
3123 mvpp2_prs_sram_bits_set(&pe, MVPP2_PRS_SRAM_LU_GEN_BIT, 1);
3124 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L3_UCAST,
3125 MVPP2_PRS_RI_L3_ADDR_MASK);
3126
3127 mvpp2_prs_tcam_ai_update(&pe, MVPP2_PRS_IPV4_DIP_AI_BIT,
3128 MVPP2_PRS_IPV4_DIP_AI_BIT);
3129 /* Unmask all ports */
3130 mvpp2_prs_tcam_port_map_set(&pe, MVPP2_PRS_PORT_MASK);
3131
3132 /* Update shadow table and hw entry */
3133 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_IP4);
3134 mvpp2_prs_hw_write(priv, &pe);
3135
3136 return 0;
3137}
3138
3139/* Initialize entries for IPv6 */
3140static int mvpp2_prs_ip6_init(struct mvpp2 *priv)
3141{
3142 struct mvpp2_prs_entry pe;
3143 int tid, err;
3144
3145 /* Set entries for TCP, UDP and ICMP over IPv6 */
3146 err = mvpp2_prs_ip6_proto(priv, IPPROTO_TCP,
3147 MVPP2_PRS_RI_L4_TCP,
3148 MVPP2_PRS_RI_L4_PROTO_MASK);
3149 if (err)
3150 return err;
3151
3152 err = mvpp2_prs_ip6_proto(priv, IPPROTO_UDP,
3153 MVPP2_PRS_RI_L4_UDP,
3154 MVPP2_PRS_RI_L4_PROTO_MASK);
3155 if (err)
3156 return err;
3157
3158 err = mvpp2_prs_ip6_proto(priv, IPPROTO_ICMPV6,
3159 MVPP2_PRS_RI_CPU_CODE_RX_SPEC |
3160 MVPP2_PRS_RI_UDF3_RX_SPECIAL,
3161 MVPP2_PRS_RI_CPU_CODE_MASK |
3162 MVPP2_PRS_RI_UDF3_MASK);
3163 if (err)
3164 return err;
3165
3166 /* IPv4 is the last header. This is similar case as 6-TCP or 17-UDP */
3167 /* Result Info: UDF7=1, DS lite */
3168 err = mvpp2_prs_ip6_proto(priv, IPPROTO_IPIP,
3169 MVPP2_PRS_RI_UDF7_IP6_LITE,
3170 MVPP2_PRS_RI_UDF7_MASK);
3171 if (err)
3172 return err;
3173
3174 /* IPv6 multicast */
3175 err = mvpp2_prs_ip6_cast(priv, MVPP2_PRS_L3_MULTI_CAST);
3176 if (err)
3177 return err;
3178
3179 /* Entry for checking hop limit */
3180 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_FIRST_FREE_TID,
3181 MVPP2_PE_LAST_FREE_TID);
3182 if (tid < 0)
3183 return tid;
3184
Markus Elfringc5b2ce22017-04-17 10:30:29 +02003185 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03003186 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_IP6);
3187 pe.index = tid;
3188
3189 /* Finished: go to flowid generation */
3190 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_FLOWS);
3191 mvpp2_prs_sram_bits_set(&pe, MVPP2_PRS_SRAM_LU_GEN_BIT, 1);
3192 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L3_UN |
3193 MVPP2_PRS_RI_DROP_MASK,
3194 MVPP2_PRS_RI_L3_PROTO_MASK |
3195 MVPP2_PRS_RI_DROP_MASK);
3196
3197 mvpp2_prs_tcam_data_byte_set(&pe, 1, 0x00, MVPP2_PRS_IPV6_HOP_MASK);
3198 mvpp2_prs_tcam_ai_update(&pe, MVPP2_PRS_IPV6_NO_EXT_AI_BIT,
3199 MVPP2_PRS_IPV6_NO_EXT_AI_BIT);
3200
3201 /* Update shadow table and hw entry */
3202 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_IP4);
3203 mvpp2_prs_hw_write(priv, &pe);
3204
3205 /* Default IPv6 entry for unknown protocols */
Markus Elfringc5b2ce22017-04-17 10:30:29 +02003206 memset(&pe, 0, sizeof(pe));
Marcin Wojtas3f518502014-07-10 16:52:13 -03003207 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_IP6);
3208 pe.index = MVPP2_PE_IP6_PROTO_UN;
3209
3210 /* Finished: go to flowid generation */
3211 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_FLOWS);
3212 mvpp2_prs_sram_bits_set(&pe, MVPP2_PRS_SRAM_LU_GEN_BIT, 1);
3213 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L4_OTHER,
3214 MVPP2_PRS_RI_L4_PROTO_MASK);
3215 /* Set L4 offset relatively to our current place */
3216 mvpp2_prs_sram_offset_set(&pe, MVPP2_PRS_SRAM_UDF_TYPE_L4,
3217 sizeof(struct ipv6hdr) - 4,
3218 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD);
3219
3220 mvpp2_prs_tcam_ai_update(&pe, MVPP2_PRS_IPV6_NO_EXT_AI_BIT,
3221 MVPP2_PRS_IPV6_NO_EXT_AI_BIT);
3222 /* Unmask all ports */
3223 mvpp2_prs_tcam_port_map_set(&pe, MVPP2_PRS_PORT_MASK);
3224
3225 /* Update shadow table and hw entry */
3226 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_IP4);
3227 mvpp2_prs_hw_write(priv, &pe);
3228
3229 /* Default IPv6 entry for unknown ext protocols */
3230 memset(&pe, 0, sizeof(struct mvpp2_prs_entry));
3231 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_IP6);
3232 pe.index = MVPP2_PE_IP6_EXT_PROTO_UN;
3233
3234 /* Finished: go to flowid generation */
3235 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_FLOWS);
3236 mvpp2_prs_sram_bits_set(&pe, MVPP2_PRS_SRAM_LU_GEN_BIT, 1);
3237 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L4_OTHER,
3238 MVPP2_PRS_RI_L4_PROTO_MASK);
3239
3240 mvpp2_prs_tcam_ai_update(&pe, MVPP2_PRS_IPV6_EXT_AI_BIT,
3241 MVPP2_PRS_IPV6_EXT_AI_BIT);
3242 /* Unmask all ports */
3243 mvpp2_prs_tcam_port_map_set(&pe, MVPP2_PRS_PORT_MASK);
3244
3245 /* Update shadow table and hw entry */
3246 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_IP4);
3247 mvpp2_prs_hw_write(priv, &pe);
3248
3249 /* Default IPv6 entry for unicast address */
3250 memset(&pe, 0, sizeof(struct mvpp2_prs_entry));
3251 mvpp2_prs_tcam_lu_set(&pe, MVPP2_PRS_LU_IP6);
3252 pe.index = MVPP2_PE_IP6_ADDR_UN;
3253
3254 /* Finished: go to IPv6 again */
3255 mvpp2_prs_sram_next_lu_set(&pe, MVPP2_PRS_LU_IP6);
3256 mvpp2_prs_sram_ri_update(&pe, MVPP2_PRS_RI_L3_UCAST,
3257 MVPP2_PRS_RI_L3_ADDR_MASK);
3258 mvpp2_prs_sram_ai_update(&pe, MVPP2_PRS_IPV6_NO_EXT_AI_BIT,
3259 MVPP2_PRS_IPV6_NO_EXT_AI_BIT);
3260 /* Shift back to IPV6 NH */
3261 mvpp2_prs_sram_shift_set(&pe, -18, MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
3262
3263 mvpp2_prs_tcam_ai_update(&pe, 0, MVPP2_PRS_IPV6_NO_EXT_AI_BIT);
3264 /* Unmask all ports */
3265 mvpp2_prs_tcam_port_map_set(&pe, MVPP2_PRS_PORT_MASK);
3266
3267 /* Update shadow table and hw entry */
3268 mvpp2_prs_shadow_set(priv, pe.index, MVPP2_PRS_LU_IP6);
3269 mvpp2_prs_hw_write(priv, &pe);
3270
3271 return 0;
3272}
3273
3274/* Parser default initialization */
3275static int mvpp2_prs_default_init(struct platform_device *pdev,
3276 struct mvpp2 *priv)
3277{
3278 int err, index, i;
3279
3280 /* Enable tcam table */
3281 mvpp2_write(priv, MVPP2_PRS_TCAM_CTRL_REG, MVPP2_PRS_TCAM_EN_MASK);
3282
3283 /* Clear all tcam and sram entries */
3284 for (index = 0; index < MVPP2_PRS_TCAM_SRAM_SIZE; index++) {
3285 mvpp2_write(priv, MVPP2_PRS_TCAM_IDX_REG, index);
3286 for (i = 0; i < MVPP2_PRS_TCAM_WORDS; i++)
3287 mvpp2_write(priv, MVPP2_PRS_TCAM_DATA_REG(i), 0);
3288
3289 mvpp2_write(priv, MVPP2_PRS_SRAM_IDX_REG, index);
3290 for (i = 0; i < MVPP2_PRS_SRAM_WORDS; i++)
3291 mvpp2_write(priv, MVPP2_PRS_SRAM_DATA_REG(i), 0);
3292 }
3293
3294 /* Invalidate all tcam entries */
3295 for (index = 0; index < MVPP2_PRS_TCAM_SRAM_SIZE; index++)
3296 mvpp2_prs_hw_inv(priv, index);
3297
3298 priv->prs_shadow = devm_kcalloc(&pdev->dev, MVPP2_PRS_TCAM_SRAM_SIZE,
Markus Elfring37df25e2017-04-17 09:12:34 +02003299 sizeof(*priv->prs_shadow),
Marcin Wojtas3f518502014-07-10 16:52:13 -03003300 GFP_KERNEL);
3301 if (!priv->prs_shadow)
3302 return -ENOMEM;
3303
3304 /* Always start from lookup = 0 */
3305 for (index = 0; index < MVPP2_MAX_PORTS; index++)
3306 mvpp2_prs_hw_port_init(priv, index, MVPP2_PRS_LU_MH,
3307 MVPP2_PRS_PORT_LU_MAX, 0);
3308
3309 mvpp2_prs_def_flow_init(priv);
3310
3311 mvpp2_prs_mh_init(priv);
3312
3313 mvpp2_prs_mac_init(priv);
3314
3315 mvpp2_prs_dsa_init(priv);
3316
3317 err = mvpp2_prs_etype_init(priv);
3318 if (err)
3319 return err;
3320
3321 err = mvpp2_prs_vlan_init(pdev, priv);
3322 if (err)
3323 return err;
3324
3325 err = mvpp2_prs_pppoe_init(priv);
3326 if (err)
3327 return err;
3328
3329 err = mvpp2_prs_ip6_init(priv);
3330 if (err)
3331 return err;
3332
3333 err = mvpp2_prs_ip4_init(priv);
3334 if (err)
3335 return err;
3336
3337 return 0;
3338}
3339
3340/* Compare MAC DA with tcam entry data */
3341static bool mvpp2_prs_mac_range_equals(struct mvpp2_prs_entry *pe,
3342 const u8 *da, unsigned char *mask)
3343{
3344 unsigned char tcam_byte, tcam_mask;
3345 int index;
3346
3347 for (index = 0; index < ETH_ALEN; index++) {
3348 mvpp2_prs_tcam_data_byte_get(pe, index, &tcam_byte, &tcam_mask);
3349 if (tcam_mask != mask[index])
3350 return false;
3351
3352 if ((tcam_mask & tcam_byte) != (da[index] & mask[index]))
3353 return false;
3354 }
3355
3356 return true;
3357}
3358
3359/* Find tcam entry with matched pair <MAC DA, port> */
3360static struct mvpp2_prs_entry *
3361mvpp2_prs_mac_da_range_find(struct mvpp2 *priv, int pmap, const u8 *da,
3362 unsigned char *mask, int udf_type)
3363{
3364 struct mvpp2_prs_entry *pe;
3365 int tid;
3366
3367 pe = kzalloc(sizeof(*pe), GFP_KERNEL);
3368 if (!pe)
3369 return NULL;
3370 mvpp2_prs_tcam_lu_set(pe, MVPP2_PRS_LU_MAC);
3371
3372 /* Go through the all entires with MVPP2_PRS_LU_MAC */
3373 for (tid = MVPP2_PE_FIRST_FREE_TID;
3374 tid <= MVPP2_PE_LAST_FREE_TID; tid++) {
3375 unsigned int entry_pmap;
3376
3377 if (!priv->prs_shadow[tid].valid ||
3378 (priv->prs_shadow[tid].lu != MVPP2_PRS_LU_MAC) ||
3379 (priv->prs_shadow[tid].udf != udf_type))
3380 continue;
3381
3382 pe->index = tid;
3383 mvpp2_prs_hw_read(priv, pe);
3384 entry_pmap = mvpp2_prs_tcam_port_map_get(pe);
3385
3386 if (mvpp2_prs_mac_range_equals(pe, da, mask) &&
3387 entry_pmap == pmap)
3388 return pe;
3389 }
3390 kfree(pe);
3391
3392 return NULL;
3393}
3394
3395/* Update parser's mac da entry */
3396static int mvpp2_prs_mac_da_accept(struct mvpp2 *priv, int port,
3397 const u8 *da, bool add)
3398{
3399 struct mvpp2_prs_entry *pe;
3400 unsigned int pmap, len, ri;
3401 unsigned char mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
3402 int tid;
3403
3404 /* Scan TCAM and see if entry with this <MAC DA, port> already exist */
3405 pe = mvpp2_prs_mac_da_range_find(priv, (1 << port), da, mask,
3406 MVPP2_PRS_UDF_MAC_DEF);
3407
3408 /* No such entry */
3409 if (!pe) {
3410 if (!add)
3411 return 0;
3412
3413 /* Create new TCAM entry */
3414 /* Find first range mac entry*/
3415 for (tid = MVPP2_PE_FIRST_FREE_TID;
3416 tid <= MVPP2_PE_LAST_FREE_TID; tid++)
3417 if (priv->prs_shadow[tid].valid &&
3418 (priv->prs_shadow[tid].lu == MVPP2_PRS_LU_MAC) &&
3419 (priv->prs_shadow[tid].udf ==
3420 MVPP2_PRS_UDF_MAC_RANGE))
3421 break;
3422
3423 /* Go through the all entries from first to last */
3424 tid = mvpp2_prs_tcam_first_free(priv, MVPP2_PE_FIRST_FREE_TID,
3425 tid - 1);
3426 if (tid < 0)
3427 return tid;
3428
3429 pe = kzalloc(sizeof(*pe), GFP_KERNEL);
3430 if (!pe)
Amitoj Kaur Chawlac2bb7bc2016-02-04 19:25:26 +05303431 return -ENOMEM;
Marcin Wojtas3f518502014-07-10 16:52:13 -03003432 mvpp2_prs_tcam_lu_set(pe, MVPP2_PRS_LU_MAC);
3433 pe->index = tid;
3434
3435 /* Mask all ports */
3436 mvpp2_prs_tcam_port_map_set(pe, 0);
3437 }
3438
3439 /* Update port mask */
3440 mvpp2_prs_tcam_port_set(pe, port, add);
3441
3442 /* Invalidate the entry if no ports are left enabled */
3443 pmap = mvpp2_prs_tcam_port_map_get(pe);
3444 if (pmap == 0) {
3445 if (add) {
3446 kfree(pe);
Amitoj Kaur Chawlac2bb7bc2016-02-04 19:25:26 +05303447 return -EINVAL;
Marcin Wojtas3f518502014-07-10 16:52:13 -03003448 }
3449 mvpp2_prs_hw_inv(priv, pe->index);
3450 priv->prs_shadow[pe->index].valid = false;
3451 kfree(pe);
3452 return 0;
3453 }
3454
3455 /* Continue - set next lookup */
3456 mvpp2_prs_sram_next_lu_set(pe, MVPP2_PRS_LU_DSA);
3457
3458 /* Set match on DA */
3459 len = ETH_ALEN;
3460 while (len--)
3461 mvpp2_prs_tcam_data_byte_set(pe, len, da[len], 0xff);
3462
3463 /* Set result info bits */
3464 if (is_broadcast_ether_addr(da))
3465 ri = MVPP2_PRS_RI_L2_BCAST;
3466 else if (is_multicast_ether_addr(da))
3467 ri = MVPP2_PRS_RI_L2_MCAST;
3468 else
3469 ri = MVPP2_PRS_RI_L2_UCAST | MVPP2_PRS_RI_MAC_ME_MASK;
3470
3471 mvpp2_prs_sram_ri_update(pe, ri, MVPP2_PRS_RI_L2_CAST_MASK |
3472 MVPP2_PRS_RI_MAC_ME_MASK);
3473 mvpp2_prs_shadow_ri_set(priv, pe->index, ri, MVPP2_PRS_RI_L2_CAST_MASK |
3474 MVPP2_PRS_RI_MAC_ME_MASK);
3475
3476 /* Shift to ethertype */
3477 mvpp2_prs_sram_shift_set(pe, 2 * ETH_ALEN,
3478 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD);
3479
3480 /* Update shadow table and hw entry */
3481 priv->prs_shadow[pe->index].udf = MVPP2_PRS_UDF_MAC_DEF;
3482 mvpp2_prs_shadow_set(priv, pe->index, MVPP2_PRS_LU_MAC);
3483 mvpp2_prs_hw_write(priv, pe);
3484
3485 kfree(pe);
3486
3487 return 0;
3488}
3489
3490static int mvpp2_prs_update_mac_da(struct net_device *dev, const u8 *da)
3491{
3492 struct mvpp2_port *port = netdev_priv(dev);
3493 int err;
3494
3495 /* Remove old parser entry */
3496 err = mvpp2_prs_mac_da_accept(port->priv, port->id, dev->dev_addr,
3497 false);
3498 if (err)
3499 return err;
3500
3501 /* Add new parser entry */
3502 err = mvpp2_prs_mac_da_accept(port->priv, port->id, da, true);
3503 if (err)
3504 return err;
3505
3506 /* Set addr in the device */
3507 ether_addr_copy(dev->dev_addr, da);
3508
3509 return 0;
3510}
3511
3512/* Delete all port's multicast simple (not range) entries */
3513static void mvpp2_prs_mcast_del_all(struct mvpp2 *priv, int port)
3514{
3515 struct mvpp2_prs_entry pe;
3516 int index, tid;
3517
3518 for (tid = MVPP2_PE_FIRST_FREE_TID;
3519 tid <= MVPP2_PE_LAST_FREE_TID; tid++) {
3520 unsigned char da[ETH_ALEN], da_mask[ETH_ALEN];
3521
3522 if (!priv->prs_shadow[tid].valid ||
3523 (priv->prs_shadow[tid].lu != MVPP2_PRS_LU_MAC) ||
3524 (priv->prs_shadow[tid].udf != MVPP2_PRS_UDF_MAC_DEF))
3525 continue;
3526
3527 /* Only simple mac entries */
3528 pe.index = tid;
3529 mvpp2_prs_hw_read(priv, &pe);
3530
3531 /* Read mac addr from entry */
3532 for (index = 0; index < ETH_ALEN; index++)
3533 mvpp2_prs_tcam_data_byte_get(&pe, index, &da[index],
3534 &da_mask[index]);
3535
3536 if (is_multicast_ether_addr(da) && !is_broadcast_ether_addr(da))
3537 /* Delete this entry */
3538 mvpp2_prs_mac_da_accept(priv, port, da, false);
3539 }
3540}
3541
3542static int mvpp2_prs_tag_mode_set(struct mvpp2 *priv, int port, int type)
3543{
3544 switch (type) {
3545 case MVPP2_TAG_TYPE_EDSA:
3546 /* Add port to EDSA entries */
3547 mvpp2_prs_dsa_tag_set(priv, port, true,
3548 MVPP2_PRS_TAGGED, MVPP2_PRS_EDSA);
3549 mvpp2_prs_dsa_tag_set(priv, port, true,
3550 MVPP2_PRS_UNTAGGED, MVPP2_PRS_EDSA);
3551 /* Remove port from DSA entries */
3552 mvpp2_prs_dsa_tag_set(priv, port, false,
3553 MVPP2_PRS_TAGGED, MVPP2_PRS_DSA);
3554 mvpp2_prs_dsa_tag_set(priv, port, false,
3555 MVPP2_PRS_UNTAGGED, MVPP2_PRS_DSA);
3556 break;
3557
3558 case MVPP2_TAG_TYPE_DSA:
3559 /* Add port to DSA entries */
3560 mvpp2_prs_dsa_tag_set(priv, port, true,
3561 MVPP2_PRS_TAGGED, MVPP2_PRS_DSA);
3562 mvpp2_prs_dsa_tag_set(priv, port, true,
3563 MVPP2_PRS_UNTAGGED, MVPP2_PRS_DSA);
3564 /* Remove port from EDSA entries */
3565 mvpp2_prs_dsa_tag_set(priv, port, false,
3566 MVPP2_PRS_TAGGED, MVPP2_PRS_EDSA);
3567 mvpp2_prs_dsa_tag_set(priv, port, false,
3568 MVPP2_PRS_UNTAGGED, MVPP2_PRS_EDSA);
3569 break;
3570
3571 case MVPP2_TAG_TYPE_MH:
3572 case MVPP2_TAG_TYPE_NONE:
3573 /* Remove port form EDSA and DSA entries */
3574 mvpp2_prs_dsa_tag_set(priv, port, false,
3575 MVPP2_PRS_TAGGED, MVPP2_PRS_DSA);
3576 mvpp2_prs_dsa_tag_set(priv, port, false,
3577 MVPP2_PRS_UNTAGGED, MVPP2_PRS_DSA);
3578 mvpp2_prs_dsa_tag_set(priv, port, false,
3579 MVPP2_PRS_TAGGED, MVPP2_PRS_EDSA);
3580 mvpp2_prs_dsa_tag_set(priv, port, false,
3581 MVPP2_PRS_UNTAGGED, MVPP2_PRS_EDSA);
3582 break;
3583
3584 default:
3585 if ((type < 0) || (type > MVPP2_TAG_TYPE_EDSA))
3586 return -EINVAL;
3587 }
3588
3589 return 0;
3590}
3591
3592/* Set prs flow for the port */
3593static int mvpp2_prs_def_flow(struct mvpp2_port *port)
3594{
3595 struct mvpp2_prs_entry *pe;
3596 int tid;
3597
3598 pe = mvpp2_prs_flow_find(port->priv, port->id);
3599
3600 /* Such entry not exist */
3601 if (!pe) {
3602 /* Go through the all entires from last to first */
3603 tid = mvpp2_prs_tcam_first_free(port->priv,
3604 MVPP2_PE_LAST_FREE_TID,
3605 MVPP2_PE_FIRST_FREE_TID);
3606 if (tid < 0)
3607 return tid;
3608
3609 pe = kzalloc(sizeof(*pe), GFP_KERNEL);
3610 if (!pe)
3611 return -ENOMEM;
3612
3613 mvpp2_prs_tcam_lu_set(pe, MVPP2_PRS_LU_FLOWS);
3614 pe->index = tid;
3615
3616 /* Set flow ID*/
3617 mvpp2_prs_sram_ai_update(pe, port->id, MVPP2_PRS_FLOW_ID_MASK);
3618 mvpp2_prs_sram_bits_set(pe, MVPP2_PRS_SRAM_LU_DONE_BIT, 1);
3619
3620 /* Update shadow table */
3621 mvpp2_prs_shadow_set(port->priv, pe->index, MVPP2_PRS_LU_FLOWS);
3622 }
3623
3624 mvpp2_prs_tcam_port_map_set(pe, (1 << port->id));
3625 mvpp2_prs_hw_write(port->priv, pe);
3626 kfree(pe);
3627
3628 return 0;
3629}
3630
3631/* Classifier configuration routines */
3632
3633/* Update classification flow table registers */
3634static void mvpp2_cls_flow_write(struct mvpp2 *priv,
3635 struct mvpp2_cls_flow_entry *fe)
3636{
3637 mvpp2_write(priv, MVPP2_CLS_FLOW_INDEX_REG, fe->index);
3638 mvpp2_write(priv, MVPP2_CLS_FLOW_TBL0_REG, fe->data[0]);
3639 mvpp2_write(priv, MVPP2_CLS_FLOW_TBL1_REG, fe->data[1]);
3640 mvpp2_write(priv, MVPP2_CLS_FLOW_TBL2_REG, fe->data[2]);
3641}
3642
3643/* Update classification lookup table register */
3644static void mvpp2_cls_lookup_write(struct mvpp2 *priv,
3645 struct mvpp2_cls_lookup_entry *le)
3646{
3647 u32 val;
3648
3649 val = (le->way << MVPP2_CLS_LKP_INDEX_WAY_OFFS) | le->lkpid;
3650 mvpp2_write(priv, MVPP2_CLS_LKP_INDEX_REG, val);
3651 mvpp2_write(priv, MVPP2_CLS_LKP_TBL_REG, le->data);
3652}
3653
3654/* Classifier default initialization */
3655static void mvpp2_cls_init(struct mvpp2 *priv)
3656{
3657 struct mvpp2_cls_lookup_entry le;
3658 struct mvpp2_cls_flow_entry fe;
3659 int index;
3660
3661 /* Enable classifier */
3662 mvpp2_write(priv, MVPP2_CLS_MODE_REG, MVPP2_CLS_MODE_ACTIVE_MASK);
3663
3664 /* Clear classifier flow table */
Arnd Bergmanne8f967c2016-11-24 17:28:12 +01003665 memset(&fe.data, 0, sizeof(fe.data));
Marcin Wojtas3f518502014-07-10 16:52:13 -03003666 for (index = 0; index < MVPP2_CLS_FLOWS_TBL_SIZE; index++) {
3667 fe.index = index;
3668 mvpp2_cls_flow_write(priv, &fe);
3669 }
3670
3671 /* Clear classifier lookup table */
3672 le.data = 0;
3673 for (index = 0; index < MVPP2_CLS_LKP_TBL_SIZE; index++) {
3674 le.lkpid = index;
3675 le.way = 0;
3676 mvpp2_cls_lookup_write(priv, &le);
3677
3678 le.way = 1;
3679 mvpp2_cls_lookup_write(priv, &le);
3680 }
3681}
3682
3683static void mvpp2_cls_port_config(struct mvpp2_port *port)
3684{
3685 struct mvpp2_cls_lookup_entry le;
3686 u32 val;
3687
3688 /* Set way for the port */
3689 val = mvpp2_read(port->priv, MVPP2_CLS_PORT_WAY_REG);
3690 val &= ~MVPP2_CLS_PORT_WAY_MASK(port->id);
3691 mvpp2_write(port->priv, MVPP2_CLS_PORT_WAY_REG, val);
3692
3693 /* Pick the entry to be accessed in lookup ID decoding table
3694 * according to the way and lkpid.
3695 */
3696 le.lkpid = port->id;
3697 le.way = 0;
3698 le.data = 0;
3699
3700 /* Set initial CPU queue for receiving packets */
3701 le.data &= ~MVPP2_CLS_LKP_TBL_RXQ_MASK;
3702 le.data |= port->first_rxq;
3703
3704 /* Disable classification engines */
3705 le.data &= ~MVPP2_CLS_LKP_TBL_LOOKUP_EN_MASK;
3706
3707 /* Update lookup ID table entry */
3708 mvpp2_cls_lookup_write(port->priv, &le);
3709}
3710
3711/* Set CPU queue number for oversize packets */
3712static void mvpp2_cls_oversize_rxq_set(struct mvpp2_port *port)
3713{
3714 u32 val;
3715
3716 mvpp2_write(port->priv, MVPP2_CLS_OVERSIZE_RXQ_LOW_REG(port->id),
3717 port->first_rxq & MVPP2_CLS_OVERSIZE_RXQ_LOW_MASK);
3718
3719 mvpp2_write(port->priv, MVPP2_CLS_SWFWD_P2HQ_REG(port->id),
3720 (port->first_rxq >> MVPP2_CLS_OVERSIZE_RXQ_LOW_BITS));
3721
3722 val = mvpp2_read(port->priv, MVPP2_CLS_SWFWD_PCTRL_REG);
3723 val |= MVPP2_CLS_SWFWD_PCTRL_MASK(port->id);
3724 mvpp2_write(port->priv, MVPP2_CLS_SWFWD_PCTRL_REG, val);
3725}
3726
Thomas Petazzoni0e037282017-02-21 11:28:12 +01003727static void *mvpp2_frag_alloc(const struct mvpp2_bm_pool *pool)
3728{
3729 if (likely(pool->frag_size <= PAGE_SIZE))
3730 return netdev_alloc_frag(pool->frag_size);
3731 else
3732 return kmalloc(pool->frag_size, GFP_ATOMIC);
3733}
3734
3735static void mvpp2_frag_free(const struct mvpp2_bm_pool *pool, void *data)
3736{
3737 if (likely(pool->frag_size <= PAGE_SIZE))
3738 skb_free_frag(data);
3739 else
3740 kfree(data);
3741}
3742
Marcin Wojtas3f518502014-07-10 16:52:13 -03003743/* Buffer Manager configuration routines */
3744
3745/* Create pool */
3746static int mvpp2_bm_pool_create(struct platform_device *pdev,
3747 struct mvpp2 *priv,
3748 struct mvpp2_bm_pool *bm_pool, int size)
3749{
Marcin Wojtas3f518502014-07-10 16:52:13 -03003750 u32 val;
3751
Thomas Petazzonid01524d2017-03-07 16:53:09 +01003752 /* Number of buffer pointers must be a multiple of 16, as per
3753 * hardware constraints
3754 */
3755 if (!IS_ALIGNED(size, 16))
3756 return -EINVAL;
3757
3758 /* PPv2.1 needs 8 bytes per buffer pointer, PPv2.2 needs 16
3759 * bytes per buffer pointer
3760 */
3761 if (priv->hw_version == MVPP21)
3762 bm_pool->size_bytes = 2 * sizeof(u32) * size;
3763 else
3764 bm_pool->size_bytes = 2 * sizeof(u64) * size;
3765
3766 bm_pool->virt_addr = dma_alloc_coherent(&pdev->dev, bm_pool->size_bytes,
Thomas Petazzoni20396132017-03-07 16:53:00 +01003767 &bm_pool->dma_addr,
Marcin Wojtas3f518502014-07-10 16:52:13 -03003768 GFP_KERNEL);
3769 if (!bm_pool->virt_addr)
3770 return -ENOMEM;
3771
Thomas Petazzonid3158802017-02-21 11:28:13 +01003772 if (!IS_ALIGNED((unsigned long)bm_pool->virt_addr,
3773 MVPP2_BM_POOL_PTR_ALIGN)) {
Thomas Petazzonid01524d2017-03-07 16:53:09 +01003774 dma_free_coherent(&pdev->dev, bm_pool->size_bytes,
3775 bm_pool->virt_addr, bm_pool->dma_addr);
Marcin Wojtas3f518502014-07-10 16:52:13 -03003776 dev_err(&pdev->dev, "BM pool %d is not %d bytes aligned\n",
3777 bm_pool->id, MVPP2_BM_POOL_PTR_ALIGN);
3778 return -ENOMEM;
3779 }
3780
3781 mvpp2_write(priv, MVPP2_BM_POOL_BASE_REG(bm_pool->id),
Thomas Petazzonid01524d2017-03-07 16:53:09 +01003782 lower_32_bits(bm_pool->dma_addr));
Marcin Wojtas3f518502014-07-10 16:52:13 -03003783 mvpp2_write(priv, MVPP2_BM_POOL_SIZE_REG(bm_pool->id), size);
3784
3785 val = mvpp2_read(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id));
3786 val |= MVPP2_BM_START_MASK;
3787 mvpp2_write(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id), val);
3788
3789 bm_pool->type = MVPP2_BM_FREE;
3790 bm_pool->size = size;
3791 bm_pool->pkt_size = 0;
3792 bm_pool->buf_num = 0;
Marcin Wojtas3f518502014-07-10 16:52:13 -03003793
3794 return 0;
3795}
3796
3797/* Set pool buffer size */
3798static void mvpp2_bm_pool_bufsize_set(struct mvpp2 *priv,
3799 struct mvpp2_bm_pool *bm_pool,
3800 int buf_size)
3801{
3802 u32 val;
3803
3804 bm_pool->buf_size = buf_size;
3805
3806 val = ALIGN(buf_size, 1 << MVPP2_POOL_BUF_SIZE_OFFSET);
3807 mvpp2_write(priv, MVPP2_POOL_BUF_SIZE_REG(bm_pool->id), val);
3808}
3809
Thomas Petazzonid01524d2017-03-07 16:53:09 +01003810static void mvpp2_bm_bufs_get_addrs(struct device *dev, struct mvpp2 *priv,
3811 struct mvpp2_bm_pool *bm_pool,
3812 dma_addr_t *dma_addr,
3813 phys_addr_t *phys_addr)
3814{
Thomas Petazzonia704bb52017-06-10 23:18:22 +02003815 int cpu = get_cpu();
Thomas Petazzonia7868412017-03-07 16:53:13 +01003816
3817 *dma_addr = mvpp2_percpu_read(priv, cpu,
3818 MVPP2_BM_PHY_ALLOC_REG(bm_pool->id));
3819 *phys_addr = mvpp2_percpu_read(priv, cpu, MVPP2_BM_VIRT_ALLOC_REG);
Thomas Petazzonid01524d2017-03-07 16:53:09 +01003820
3821 if (priv->hw_version == MVPP22) {
3822 u32 val;
3823 u32 dma_addr_highbits, phys_addr_highbits;
3824
Thomas Petazzonia7868412017-03-07 16:53:13 +01003825 val = mvpp2_percpu_read(priv, cpu, MVPP22_BM_ADDR_HIGH_ALLOC);
Thomas Petazzonid01524d2017-03-07 16:53:09 +01003826 dma_addr_highbits = (val & MVPP22_BM_ADDR_HIGH_PHYS_MASK);
3827 phys_addr_highbits = (val & MVPP22_BM_ADDR_HIGH_VIRT_MASK) >>
3828 MVPP22_BM_ADDR_HIGH_VIRT_SHIFT;
3829
3830 if (sizeof(dma_addr_t) == 8)
3831 *dma_addr |= (u64)dma_addr_highbits << 32;
3832
3833 if (sizeof(phys_addr_t) == 8)
3834 *phys_addr |= (u64)phys_addr_highbits << 32;
3835 }
Thomas Petazzonia704bb52017-06-10 23:18:22 +02003836
3837 put_cpu();
Thomas Petazzonid01524d2017-03-07 16:53:09 +01003838}
3839
Ezequiel Garcia7861f122014-07-21 13:48:14 -03003840/* Free all buffers from the pool */
Marcin Wojtas4229d502015-12-03 15:20:50 +01003841static void mvpp2_bm_bufs_free(struct device *dev, struct mvpp2 *priv,
3842 struct mvpp2_bm_pool *bm_pool)
Marcin Wojtas3f518502014-07-10 16:52:13 -03003843{
3844 int i;
3845
Ezequiel Garcia7861f122014-07-21 13:48:14 -03003846 for (i = 0; i < bm_pool->buf_num; i++) {
Thomas Petazzoni20396132017-03-07 16:53:00 +01003847 dma_addr_t buf_dma_addr;
Thomas Petazzoni4e4a1052017-03-07 16:53:04 +01003848 phys_addr_t buf_phys_addr;
3849 void *data;
Marcin Wojtas3f518502014-07-10 16:52:13 -03003850
Thomas Petazzonid01524d2017-03-07 16:53:09 +01003851 mvpp2_bm_bufs_get_addrs(dev, priv, bm_pool,
3852 &buf_dma_addr, &buf_phys_addr);
Marcin Wojtas4229d502015-12-03 15:20:50 +01003853
Thomas Petazzoni20396132017-03-07 16:53:00 +01003854 dma_unmap_single(dev, buf_dma_addr,
Marcin Wojtas4229d502015-12-03 15:20:50 +01003855 bm_pool->buf_size, DMA_FROM_DEVICE);
3856
Thomas Petazzoni4e4a1052017-03-07 16:53:04 +01003857 data = (void *)phys_to_virt(buf_phys_addr);
3858 if (!data)
Marcin Wojtas3f518502014-07-10 16:52:13 -03003859 break;
Thomas Petazzoni0e037282017-02-21 11:28:12 +01003860
Thomas Petazzoni4e4a1052017-03-07 16:53:04 +01003861 mvpp2_frag_free(bm_pool, data);
Marcin Wojtas3f518502014-07-10 16:52:13 -03003862 }
3863
3864 /* Update BM driver with number of buffers removed from pool */
3865 bm_pool->buf_num -= i;
Marcin Wojtas3f518502014-07-10 16:52:13 -03003866}
3867
3868/* Cleanup pool */
3869static int mvpp2_bm_pool_destroy(struct platform_device *pdev,
3870 struct mvpp2 *priv,
3871 struct mvpp2_bm_pool *bm_pool)
3872{
Marcin Wojtas3f518502014-07-10 16:52:13 -03003873 u32 val;
3874
Marcin Wojtas4229d502015-12-03 15:20:50 +01003875 mvpp2_bm_bufs_free(&pdev->dev, priv, bm_pool);
Ezequiel Garciad74c96c2014-07-21 13:48:13 -03003876 if (bm_pool->buf_num) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03003877 WARN(1, "cannot free all buffers in pool %d\n", bm_pool->id);
3878 return 0;
3879 }
3880
3881 val = mvpp2_read(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id));
3882 val |= MVPP2_BM_STOP_MASK;
3883 mvpp2_write(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id), val);
3884
Thomas Petazzonid01524d2017-03-07 16:53:09 +01003885 dma_free_coherent(&pdev->dev, bm_pool->size_bytes,
Marcin Wojtas3f518502014-07-10 16:52:13 -03003886 bm_pool->virt_addr,
Thomas Petazzoni20396132017-03-07 16:53:00 +01003887 bm_pool->dma_addr);
Marcin Wojtas3f518502014-07-10 16:52:13 -03003888 return 0;
3889}
3890
3891static int mvpp2_bm_pools_init(struct platform_device *pdev,
3892 struct mvpp2 *priv)
3893{
3894 int i, err, size;
3895 struct mvpp2_bm_pool *bm_pool;
3896
3897 /* Create all pools with maximum size */
3898 size = MVPP2_BM_POOL_SIZE_MAX;
3899 for (i = 0; i < MVPP2_BM_POOLS_NUM; i++) {
3900 bm_pool = &priv->bm_pools[i];
3901 bm_pool->id = i;
3902 err = mvpp2_bm_pool_create(pdev, priv, bm_pool, size);
3903 if (err)
3904 goto err_unroll_pools;
3905 mvpp2_bm_pool_bufsize_set(priv, bm_pool, 0);
3906 }
3907 return 0;
3908
3909err_unroll_pools:
3910 dev_err(&pdev->dev, "failed to create BM pool %d, size %d\n", i, size);
3911 for (i = i - 1; i >= 0; i--)
3912 mvpp2_bm_pool_destroy(pdev, priv, &priv->bm_pools[i]);
3913 return err;
3914}
3915
3916static int mvpp2_bm_init(struct platform_device *pdev, struct mvpp2 *priv)
3917{
3918 int i, err;
3919
3920 for (i = 0; i < MVPP2_BM_POOLS_NUM; i++) {
3921 /* Mask BM all interrupts */
3922 mvpp2_write(priv, MVPP2_BM_INTR_MASK_REG(i), 0);
3923 /* Clear BM cause register */
3924 mvpp2_write(priv, MVPP2_BM_INTR_CAUSE_REG(i), 0);
3925 }
3926
3927 /* Allocate and initialize BM pools */
3928 priv->bm_pools = devm_kcalloc(&pdev->dev, MVPP2_BM_POOLS_NUM,
Markus Elfring81f915e2017-04-17 09:06:33 +02003929 sizeof(*priv->bm_pools), GFP_KERNEL);
Marcin Wojtas3f518502014-07-10 16:52:13 -03003930 if (!priv->bm_pools)
3931 return -ENOMEM;
3932
3933 err = mvpp2_bm_pools_init(pdev, priv);
3934 if (err < 0)
3935 return err;
3936 return 0;
3937}
3938
3939/* Attach long pool to rxq */
3940static void mvpp2_rxq_long_pool_set(struct mvpp2_port *port,
3941 int lrxq, int long_pool)
3942{
Thomas Petazzoni5eac8922017-03-07 16:53:10 +01003943 u32 val, mask;
Marcin Wojtas3f518502014-07-10 16:52:13 -03003944 int prxq;
3945
3946 /* Get queue physical ID */
3947 prxq = port->rxqs[lrxq]->id;
3948
Thomas Petazzoni5eac8922017-03-07 16:53:10 +01003949 if (port->priv->hw_version == MVPP21)
3950 mask = MVPP21_RXQ_POOL_LONG_MASK;
3951 else
3952 mask = MVPP22_RXQ_POOL_LONG_MASK;
Marcin Wojtas3f518502014-07-10 16:52:13 -03003953
Thomas Petazzoni5eac8922017-03-07 16:53:10 +01003954 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(prxq));
3955 val &= ~mask;
3956 val |= (long_pool << MVPP2_RXQ_POOL_LONG_OFFS) & mask;
Marcin Wojtas3f518502014-07-10 16:52:13 -03003957 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(prxq), val);
3958}
3959
3960/* Attach short pool to rxq */
3961static void mvpp2_rxq_short_pool_set(struct mvpp2_port *port,
3962 int lrxq, int short_pool)
3963{
Thomas Petazzoni5eac8922017-03-07 16:53:10 +01003964 u32 val, mask;
Marcin Wojtas3f518502014-07-10 16:52:13 -03003965 int prxq;
3966
3967 /* Get queue physical ID */
3968 prxq = port->rxqs[lrxq]->id;
3969
Thomas Petazzoni5eac8922017-03-07 16:53:10 +01003970 if (port->priv->hw_version == MVPP21)
3971 mask = MVPP21_RXQ_POOL_SHORT_MASK;
3972 else
3973 mask = MVPP22_RXQ_POOL_SHORT_MASK;
Marcin Wojtas3f518502014-07-10 16:52:13 -03003974
Thomas Petazzoni5eac8922017-03-07 16:53:10 +01003975 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(prxq));
3976 val &= ~mask;
3977 val |= (short_pool << MVPP2_RXQ_POOL_SHORT_OFFS) & mask;
Marcin Wojtas3f518502014-07-10 16:52:13 -03003978 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(prxq), val);
3979}
3980
Thomas Petazzoni0e037282017-02-21 11:28:12 +01003981static void *mvpp2_buf_alloc(struct mvpp2_port *port,
3982 struct mvpp2_bm_pool *bm_pool,
Thomas Petazzoni20396132017-03-07 16:53:00 +01003983 dma_addr_t *buf_dma_addr,
Thomas Petazzoni4e4a1052017-03-07 16:53:04 +01003984 phys_addr_t *buf_phys_addr,
Thomas Petazzoni0e037282017-02-21 11:28:12 +01003985 gfp_t gfp_mask)
Marcin Wojtas3f518502014-07-10 16:52:13 -03003986{
Thomas Petazzoni20396132017-03-07 16:53:00 +01003987 dma_addr_t dma_addr;
Thomas Petazzoni0e037282017-02-21 11:28:12 +01003988 void *data;
Marcin Wojtas3f518502014-07-10 16:52:13 -03003989
Thomas Petazzoni0e037282017-02-21 11:28:12 +01003990 data = mvpp2_frag_alloc(bm_pool);
3991 if (!data)
Marcin Wojtas3f518502014-07-10 16:52:13 -03003992 return NULL;
3993
Thomas Petazzoni20396132017-03-07 16:53:00 +01003994 dma_addr = dma_map_single(port->dev->dev.parent, data,
3995 MVPP2_RX_BUF_SIZE(bm_pool->pkt_size),
3996 DMA_FROM_DEVICE);
3997 if (unlikely(dma_mapping_error(port->dev->dev.parent, dma_addr))) {
Thomas Petazzoni0e037282017-02-21 11:28:12 +01003998 mvpp2_frag_free(bm_pool, data);
Marcin Wojtas3f518502014-07-10 16:52:13 -03003999 return NULL;
4000 }
Thomas Petazzoni20396132017-03-07 16:53:00 +01004001 *buf_dma_addr = dma_addr;
Thomas Petazzoni4e4a1052017-03-07 16:53:04 +01004002 *buf_phys_addr = virt_to_phys(data);
Marcin Wojtas3f518502014-07-10 16:52:13 -03004003
Thomas Petazzoni0e037282017-02-21 11:28:12 +01004004 return data;
Marcin Wojtas3f518502014-07-10 16:52:13 -03004005}
4006
Marcin Wojtas3f518502014-07-10 16:52:13 -03004007/* Release buffer to BM */
4008static inline void mvpp2_bm_pool_put(struct mvpp2_port *port, int pool,
Thomas Petazzoni20396132017-03-07 16:53:00 +01004009 dma_addr_t buf_dma_addr,
Thomas Petazzoni4e4a1052017-03-07 16:53:04 +01004010 phys_addr_t buf_phys_addr)
Marcin Wojtas3f518502014-07-10 16:52:13 -03004011{
Thomas Petazzonia704bb52017-06-10 23:18:22 +02004012 int cpu = get_cpu();
Thomas Petazzonia7868412017-03-07 16:53:13 +01004013
Thomas Petazzonid01524d2017-03-07 16:53:09 +01004014 if (port->priv->hw_version == MVPP22) {
4015 u32 val = 0;
4016
4017 if (sizeof(dma_addr_t) == 8)
4018 val |= upper_32_bits(buf_dma_addr) &
4019 MVPP22_BM_ADDR_HIGH_PHYS_RLS_MASK;
4020
4021 if (sizeof(phys_addr_t) == 8)
4022 val |= (upper_32_bits(buf_phys_addr)
4023 << MVPP22_BM_ADDR_HIGH_VIRT_RLS_SHIFT) &
4024 MVPP22_BM_ADDR_HIGH_VIRT_RLS_MASK;
4025
Thomas Petazzonia7868412017-03-07 16:53:13 +01004026 mvpp2_percpu_write(port->priv, cpu,
4027 MVPP22_BM_ADDR_HIGH_RLS_REG, val);
Thomas Petazzonid01524d2017-03-07 16:53:09 +01004028 }
4029
Thomas Petazzoni4e4a1052017-03-07 16:53:04 +01004030 /* MVPP2_BM_VIRT_RLS_REG is not interpreted by HW, and simply
4031 * returned in the "cookie" field of the RX
4032 * descriptor. Instead of storing the virtual address, we
4033 * store the physical address
4034 */
Thomas Petazzonia7868412017-03-07 16:53:13 +01004035 mvpp2_percpu_write(port->priv, cpu,
4036 MVPP2_BM_VIRT_RLS_REG, buf_phys_addr);
4037 mvpp2_percpu_write(port->priv, cpu,
4038 MVPP2_BM_PHY_RLS_REG(pool), buf_dma_addr);
Thomas Petazzonia704bb52017-06-10 23:18:22 +02004039
4040 put_cpu();
Marcin Wojtas3f518502014-07-10 16:52:13 -03004041}
4042
Marcin Wojtas3f518502014-07-10 16:52:13 -03004043/* Allocate buffers for the pool */
4044static int mvpp2_bm_bufs_add(struct mvpp2_port *port,
4045 struct mvpp2_bm_pool *bm_pool, int buf_num)
4046{
Marcin Wojtas3f518502014-07-10 16:52:13 -03004047 int i, buf_size, total_size;
Thomas Petazzoni20396132017-03-07 16:53:00 +01004048 dma_addr_t dma_addr;
Thomas Petazzoni4e4a1052017-03-07 16:53:04 +01004049 phys_addr_t phys_addr;
Thomas Petazzoni0e037282017-02-21 11:28:12 +01004050 void *buf;
Marcin Wojtas3f518502014-07-10 16:52:13 -03004051
4052 buf_size = MVPP2_RX_BUF_SIZE(bm_pool->pkt_size);
4053 total_size = MVPP2_RX_TOTAL_SIZE(buf_size);
4054
4055 if (buf_num < 0 ||
4056 (buf_num + bm_pool->buf_num > bm_pool->size)) {
4057 netdev_err(port->dev,
4058 "cannot allocate %d buffers for pool %d\n",
4059 buf_num, bm_pool->id);
4060 return 0;
4061 }
4062
Marcin Wojtas3f518502014-07-10 16:52:13 -03004063 for (i = 0; i < buf_num; i++) {
Thomas Petazzoni4e4a1052017-03-07 16:53:04 +01004064 buf = mvpp2_buf_alloc(port, bm_pool, &dma_addr,
4065 &phys_addr, GFP_KERNEL);
Thomas Petazzoni0e037282017-02-21 11:28:12 +01004066 if (!buf)
Marcin Wojtas3f518502014-07-10 16:52:13 -03004067 break;
4068
Thomas Petazzoni20396132017-03-07 16:53:00 +01004069 mvpp2_bm_pool_put(port, bm_pool->id, dma_addr,
Thomas Petazzoni4e4a1052017-03-07 16:53:04 +01004070 phys_addr);
Marcin Wojtas3f518502014-07-10 16:52:13 -03004071 }
4072
4073 /* Update BM driver with number of buffers added to pool */
4074 bm_pool->buf_num += i;
Marcin Wojtas3f518502014-07-10 16:52:13 -03004075
4076 netdev_dbg(port->dev,
4077 "%s pool %d: pkt_size=%4d, buf_size=%4d, total_size=%4d\n",
4078 bm_pool->type == MVPP2_BM_SWF_SHORT ? "short" : " long",
4079 bm_pool->id, bm_pool->pkt_size, buf_size, total_size);
4080
4081 netdev_dbg(port->dev,
4082 "%s pool %d: %d of %d buffers added\n",
4083 bm_pool->type == MVPP2_BM_SWF_SHORT ? "short" : " long",
4084 bm_pool->id, i, buf_num);
4085 return i;
4086}
4087
4088/* Notify the driver that BM pool is being used as specific type and return the
4089 * pool pointer on success
4090 */
4091static struct mvpp2_bm_pool *
4092mvpp2_bm_pool_use(struct mvpp2_port *port, int pool, enum mvpp2_bm_type type,
4093 int pkt_size)
4094{
Marcin Wojtas3f518502014-07-10 16:52:13 -03004095 struct mvpp2_bm_pool *new_pool = &port->priv->bm_pools[pool];
4096 int num;
4097
4098 if (new_pool->type != MVPP2_BM_FREE && new_pool->type != type) {
4099 netdev_err(port->dev, "mixing pool types is forbidden\n");
4100 return NULL;
4101 }
4102
Marcin Wojtas3f518502014-07-10 16:52:13 -03004103 if (new_pool->type == MVPP2_BM_FREE)
4104 new_pool->type = type;
4105
4106 /* Allocate buffers in case BM pool is used as long pool, but packet
4107 * size doesn't match MTU or BM pool hasn't being used yet
4108 */
4109 if (((type == MVPP2_BM_SWF_LONG) && (pkt_size > new_pool->pkt_size)) ||
4110 (new_pool->pkt_size == 0)) {
4111 int pkts_num;
4112
4113 /* Set default buffer number or free all the buffers in case
4114 * the pool is not empty
4115 */
4116 pkts_num = new_pool->buf_num;
4117 if (pkts_num == 0)
4118 pkts_num = type == MVPP2_BM_SWF_LONG ?
4119 MVPP2_BM_LONG_BUF_NUM :
4120 MVPP2_BM_SHORT_BUF_NUM;
4121 else
Marcin Wojtas4229d502015-12-03 15:20:50 +01004122 mvpp2_bm_bufs_free(port->dev->dev.parent,
4123 port->priv, new_pool);
Marcin Wojtas3f518502014-07-10 16:52:13 -03004124
4125 new_pool->pkt_size = pkt_size;
Thomas Petazzoni0e037282017-02-21 11:28:12 +01004126 new_pool->frag_size =
4127 SKB_DATA_ALIGN(MVPP2_RX_BUF_SIZE(pkt_size)) +
4128 MVPP2_SKB_SHINFO_SIZE;
Marcin Wojtas3f518502014-07-10 16:52:13 -03004129
4130 /* Allocate buffers for this pool */
4131 num = mvpp2_bm_bufs_add(port, new_pool, pkts_num);
4132 if (num != pkts_num) {
4133 WARN(1, "pool %d: %d of %d allocated\n",
4134 new_pool->id, num, pkts_num);
Marcin Wojtas3f518502014-07-10 16:52:13 -03004135 return NULL;
4136 }
4137 }
4138
4139 mvpp2_bm_pool_bufsize_set(port->priv, new_pool,
4140 MVPP2_RX_BUF_SIZE(new_pool->pkt_size));
4141
Marcin Wojtas3f518502014-07-10 16:52:13 -03004142 return new_pool;
4143}
4144
4145/* Initialize pools for swf */
4146static int mvpp2_swf_bm_pool_init(struct mvpp2_port *port)
4147{
Marcin Wojtas3f518502014-07-10 16:52:13 -03004148 int rxq;
4149
4150 if (!port->pool_long) {
4151 port->pool_long =
4152 mvpp2_bm_pool_use(port, MVPP2_BM_SWF_LONG_POOL(port->id),
4153 MVPP2_BM_SWF_LONG,
4154 port->pkt_size);
4155 if (!port->pool_long)
4156 return -ENOMEM;
4157
Marcin Wojtas3f518502014-07-10 16:52:13 -03004158 port->pool_long->port_map |= (1 << port->id);
Marcin Wojtas3f518502014-07-10 16:52:13 -03004159
Thomas Petazzoni09f83972017-08-03 10:41:57 +02004160 for (rxq = 0; rxq < port->nrxqs; rxq++)
Marcin Wojtas3f518502014-07-10 16:52:13 -03004161 mvpp2_rxq_long_pool_set(port, rxq, port->pool_long->id);
4162 }
4163
4164 if (!port->pool_short) {
4165 port->pool_short =
4166 mvpp2_bm_pool_use(port, MVPP2_BM_SWF_SHORT_POOL,
4167 MVPP2_BM_SWF_SHORT,
4168 MVPP2_BM_SHORT_PKT_SIZE);
4169 if (!port->pool_short)
4170 return -ENOMEM;
4171
Marcin Wojtas3f518502014-07-10 16:52:13 -03004172 port->pool_short->port_map |= (1 << port->id);
Marcin Wojtas3f518502014-07-10 16:52:13 -03004173
Thomas Petazzoni09f83972017-08-03 10:41:57 +02004174 for (rxq = 0; rxq < port->nrxqs; rxq++)
Marcin Wojtas3f518502014-07-10 16:52:13 -03004175 mvpp2_rxq_short_pool_set(port, rxq,
4176 port->pool_short->id);
4177 }
4178
4179 return 0;
4180}
4181
4182static int mvpp2_bm_update_mtu(struct net_device *dev, int mtu)
4183{
4184 struct mvpp2_port *port = netdev_priv(dev);
4185 struct mvpp2_bm_pool *port_pool = port->pool_long;
4186 int num, pkts_num = port_pool->buf_num;
4187 int pkt_size = MVPP2_RX_PKT_SIZE(mtu);
4188
4189 /* Update BM pool with new buffer size */
Marcin Wojtas4229d502015-12-03 15:20:50 +01004190 mvpp2_bm_bufs_free(dev->dev.parent, port->priv, port_pool);
Ezequiel Garciad74c96c2014-07-21 13:48:13 -03004191 if (port_pool->buf_num) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03004192 WARN(1, "cannot free all buffers in pool %d\n", port_pool->id);
4193 return -EIO;
4194 }
4195
4196 port_pool->pkt_size = pkt_size;
Thomas Petazzoni0e037282017-02-21 11:28:12 +01004197 port_pool->frag_size = SKB_DATA_ALIGN(MVPP2_RX_BUF_SIZE(pkt_size)) +
4198 MVPP2_SKB_SHINFO_SIZE;
Marcin Wojtas3f518502014-07-10 16:52:13 -03004199 num = mvpp2_bm_bufs_add(port, port_pool, pkts_num);
4200 if (num != pkts_num) {
4201 WARN(1, "pool %d: %d of %d allocated\n",
4202 port_pool->id, num, pkts_num);
4203 return -EIO;
4204 }
4205
4206 mvpp2_bm_pool_bufsize_set(port->priv, port_pool,
4207 MVPP2_RX_BUF_SIZE(port_pool->pkt_size));
4208 dev->mtu = mtu;
4209 netdev_update_features(dev);
4210 return 0;
4211}
4212
4213static inline void mvpp2_interrupts_enable(struct mvpp2_port *port)
4214{
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02004215 int i, sw_thread_mask = 0;
Marcin Wojtas3f518502014-07-10 16:52:13 -03004216
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02004217 for (i = 0; i < port->nqvecs; i++)
4218 sw_thread_mask |= port->qvecs[i].sw_thread_mask;
4219
Marcin Wojtas3f518502014-07-10 16:52:13 -03004220 mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id),
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02004221 MVPP2_ISR_ENABLE_INTERRUPT(sw_thread_mask));
Marcin Wojtas3f518502014-07-10 16:52:13 -03004222}
4223
4224static inline void mvpp2_interrupts_disable(struct mvpp2_port *port)
4225{
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02004226 int i, sw_thread_mask = 0;
Marcin Wojtas3f518502014-07-10 16:52:13 -03004227
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02004228 for (i = 0; i < port->nqvecs; i++)
4229 sw_thread_mask |= port->qvecs[i].sw_thread_mask;
4230
Marcin Wojtas3f518502014-07-10 16:52:13 -03004231 mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id),
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02004232 MVPP2_ISR_DISABLE_INTERRUPT(sw_thread_mask));
4233}
4234
4235static inline void mvpp2_qvec_interrupt_enable(struct mvpp2_queue_vector *qvec)
4236{
4237 struct mvpp2_port *port = qvec->port;
4238
4239 mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id),
4240 MVPP2_ISR_ENABLE_INTERRUPT(qvec->sw_thread_mask));
4241}
4242
4243static inline void mvpp2_qvec_interrupt_disable(struct mvpp2_queue_vector *qvec)
4244{
4245 struct mvpp2_port *port = qvec->port;
4246
4247 mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id),
4248 MVPP2_ISR_DISABLE_INTERRUPT(qvec->sw_thread_mask));
Marcin Wojtas3f518502014-07-10 16:52:13 -03004249}
4250
Thomas Petazzonie0af22d2017-06-22 14:23:18 +02004251/* Mask the current CPU's Rx/Tx interrupts
4252 * Called by on_each_cpu(), guaranteed to run with migration disabled,
4253 * using smp_processor_id() is OK.
4254 */
Marcin Wojtas3f518502014-07-10 16:52:13 -03004255static void mvpp2_interrupts_mask(void *arg)
4256{
4257 struct mvpp2_port *port = arg;
4258
Thomas Petazzonia7868412017-03-07 16:53:13 +01004259 mvpp2_percpu_write(port->priv, smp_processor_id(),
4260 MVPP2_ISR_RX_TX_MASK_REG(port->id), 0);
Marcin Wojtas3f518502014-07-10 16:52:13 -03004261}
4262
Thomas Petazzonie0af22d2017-06-22 14:23:18 +02004263/* Unmask the current CPU's Rx/Tx interrupts.
4264 * Called by on_each_cpu(), guaranteed to run with migration disabled,
4265 * using smp_processor_id() is OK.
4266 */
Marcin Wojtas3f518502014-07-10 16:52:13 -03004267static void mvpp2_interrupts_unmask(void *arg)
4268{
4269 struct mvpp2_port *port = arg;
Thomas Petazzoni213f4282017-08-03 10:42:00 +02004270 u32 val;
4271
4272 val = MVPP2_CAUSE_MISC_SUM_MASK |
4273 MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK;
4274 if (port->has_tx_irqs)
4275 val |= MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_MASK;
Marcin Wojtas3f518502014-07-10 16:52:13 -03004276
Thomas Petazzonia7868412017-03-07 16:53:13 +01004277 mvpp2_percpu_write(port->priv, smp_processor_id(),
Thomas Petazzoni213f4282017-08-03 10:42:00 +02004278 MVPP2_ISR_RX_TX_MASK_REG(port->id), val);
4279}
4280
4281static void
4282mvpp2_shared_interrupt_mask_unmask(struct mvpp2_port *port, bool mask)
4283{
4284 u32 val;
4285 int i;
4286
4287 if (port->priv->hw_version != MVPP22)
4288 return;
4289
4290 if (mask)
4291 val = 0;
4292 else
4293 val = MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK;
4294
4295 for (i = 0; i < port->nqvecs; i++) {
4296 struct mvpp2_queue_vector *v = port->qvecs + i;
4297
4298 if (v->type != MVPP2_QUEUE_VECTOR_SHARED)
4299 continue;
4300
4301 mvpp2_percpu_write(port->priv, v->sw_thread_id,
4302 MVPP2_ISR_RX_TX_MASK_REG(port->id), val);
4303 }
Marcin Wojtas3f518502014-07-10 16:52:13 -03004304}
4305
4306/* Port configuration routines */
4307
Antoine Ténartf84bf382017-08-22 19:08:27 +02004308static void mvpp22_gop_init_rgmii(struct mvpp2_port *port)
4309{
4310 struct mvpp2 *priv = port->priv;
4311 u32 val;
4312
4313 regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL0, &val);
4314 val |= GENCONF_PORT_CTRL0_BUS_WIDTH_SELECT;
4315 regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL0, val);
4316
4317 regmap_read(priv->sysctrl_base, GENCONF_CTRL0, &val);
4318 if (port->gop_id == 2)
4319 val |= GENCONF_CTRL0_PORT0_RGMII | GENCONF_CTRL0_PORT1_RGMII;
4320 else if (port->gop_id == 3)
4321 val |= GENCONF_CTRL0_PORT1_RGMII_MII;
4322 regmap_write(priv->sysctrl_base, GENCONF_CTRL0, val);
4323}
4324
4325static void mvpp22_gop_init_sgmii(struct mvpp2_port *port)
4326{
4327 struct mvpp2 *priv = port->priv;
4328 u32 val;
4329
4330 regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL0, &val);
4331 val |= GENCONF_PORT_CTRL0_BUS_WIDTH_SELECT |
4332 GENCONF_PORT_CTRL0_RX_DATA_SAMPLE;
4333 regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL0, val);
4334
4335 if (port->gop_id > 1) {
4336 regmap_read(priv->sysctrl_base, GENCONF_CTRL0, &val);
4337 if (port->gop_id == 2)
4338 val &= ~GENCONF_CTRL0_PORT0_RGMII;
4339 else if (port->gop_id == 3)
4340 val &= ~GENCONF_CTRL0_PORT1_RGMII_MII;
4341 regmap_write(priv->sysctrl_base, GENCONF_CTRL0, val);
4342 }
4343}
4344
4345static void mvpp22_gop_init_10gkr(struct mvpp2_port *port)
4346{
4347 struct mvpp2 *priv = port->priv;
4348 void __iomem *mpcs = priv->iface_base + MVPP22_MPCS_BASE(port->gop_id);
4349 void __iomem *xpcs = priv->iface_base + MVPP22_XPCS_BASE(port->gop_id);
4350 u32 val;
4351
4352 /* XPCS */
4353 val = readl(xpcs + MVPP22_XPCS_CFG0);
4354 val &= ~(MVPP22_XPCS_CFG0_PCS_MODE(0x3) |
4355 MVPP22_XPCS_CFG0_ACTIVE_LANE(0x3));
4356 val |= MVPP22_XPCS_CFG0_ACTIVE_LANE(2);
4357 writel(val, xpcs + MVPP22_XPCS_CFG0);
4358
4359 /* MPCS */
4360 val = readl(mpcs + MVPP22_MPCS_CTRL);
4361 val &= ~MVPP22_MPCS_CTRL_FWD_ERR_CONN;
4362 writel(val, mpcs + MVPP22_MPCS_CTRL);
4363
4364 val = readl(mpcs + MVPP22_MPCS_CLK_RESET);
4365 val &= ~(MVPP22_MPCS_CLK_RESET_DIV_RATIO(0x7) | MAC_CLK_RESET_MAC |
4366 MAC_CLK_RESET_SD_RX | MAC_CLK_RESET_SD_TX);
4367 val |= MVPP22_MPCS_CLK_RESET_DIV_RATIO(1);
4368 writel(val, mpcs + MVPP22_MPCS_CLK_RESET);
4369
4370 val &= ~MVPP22_MPCS_CLK_RESET_DIV_SET;
4371 val |= MAC_CLK_RESET_MAC | MAC_CLK_RESET_SD_RX | MAC_CLK_RESET_SD_TX;
4372 writel(val, mpcs + MVPP22_MPCS_CLK_RESET);
4373}
4374
4375static int mvpp22_gop_init(struct mvpp2_port *port)
4376{
4377 struct mvpp2 *priv = port->priv;
4378 u32 val;
4379
4380 if (!priv->sysctrl_base)
4381 return 0;
4382
4383 switch (port->phy_interface) {
4384 case PHY_INTERFACE_MODE_RGMII:
4385 case PHY_INTERFACE_MODE_RGMII_ID:
4386 case PHY_INTERFACE_MODE_RGMII_RXID:
4387 case PHY_INTERFACE_MODE_RGMII_TXID:
4388 if (port->gop_id == 0)
4389 goto invalid_conf;
4390 mvpp22_gop_init_rgmii(port);
4391 break;
4392 case PHY_INTERFACE_MODE_SGMII:
4393 mvpp22_gop_init_sgmii(port);
4394 break;
4395 case PHY_INTERFACE_MODE_10GKR:
4396 if (port->gop_id != 0)
4397 goto invalid_conf;
4398 mvpp22_gop_init_10gkr(port);
4399 break;
4400 default:
4401 goto unsupported_conf;
4402 }
4403
4404 regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL1, &val);
4405 val |= GENCONF_PORT_CTRL1_RESET(port->gop_id) |
4406 GENCONF_PORT_CTRL1_EN(port->gop_id);
4407 regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL1, val);
4408
4409 regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL0, &val);
4410 val |= GENCONF_PORT_CTRL0_CLK_DIV_PHASE_CLR;
4411 regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL0, val);
4412
4413 regmap_read(priv->sysctrl_base, GENCONF_SOFT_RESET1, &val);
4414 val |= GENCONF_SOFT_RESET1_GOP;
4415 regmap_write(priv->sysctrl_base, GENCONF_SOFT_RESET1, val);
4416
4417unsupported_conf:
4418 return 0;
4419
4420invalid_conf:
4421 netdev_err(port->dev, "Invalid port configuration\n");
4422 return -EINVAL;
4423}
4424
Antoine Tenart542897d2017-08-30 10:29:15 +02004425static int mvpp22_comphy_init(struct mvpp2_port *port)
4426{
4427 enum phy_mode mode;
4428 int ret;
4429
4430 if (!port->comphy)
4431 return 0;
4432
4433 switch (port->phy_interface) {
4434 case PHY_INTERFACE_MODE_SGMII:
4435 mode = PHY_MODE_SGMII;
4436 break;
4437 case PHY_INTERFACE_MODE_10GKR:
4438 mode = PHY_MODE_10GKR;
4439 break;
4440 default:
4441 return -EINVAL;
4442 }
4443
4444 ret = phy_set_mode(port->comphy, mode);
4445 if (ret)
4446 return ret;
4447
4448 return phy_power_on(port->comphy);
4449}
4450
Antoine Ténart39193572017-08-22 19:08:24 +02004451static void mvpp2_port_mii_gmac_configure_mode(struct mvpp2_port *port)
4452{
4453 u32 val;
4454
4455 if (port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
4456 val = readl(port->base + MVPP22_GMAC_CTRL_4_REG);
4457 val |= MVPP22_CTRL4_SYNC_BYPASS_DIS | MVPP22_CTRL4_DP_CLK_SEL |
4458 MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE;
4459 val &= ~MVPP22_CTRL4_EXT_PIN_GMII_SEL;
4460 writel(val, port->base + MVPP22_GMAC_CTRL_4_REG);
4461
4462 val = readl(port->base + MVPP2_GMAC_CTRL_2_REG);
4463 val |= MVPP2_GMAC_DISABLE_PADDING;
4464 val &= ~MVPP2_GMAC_FLOW_CTRL_MASK;
4465 writel(val, port->base + MVPP2_GMAC_CTRL_2_REG);
Antoine Tenart1df22702017-09-01 11:04:52 +02004466 } else if (phy_interface_mode_is_rgmii(port->phy_interface)) {
Antoine Ténart39193572017-08-22 19:08:24 +02004467 val = readl(port->base + MVPP22_GMAC_CTRL_4_REG);
4468 val |= MVPP22_CTRL4_EXT_PIN_GMII_SEL |
4469 MVPP22_CTRL4_SYNC_BYPASS_DIS |
4470 MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE;
4471 val &= ~MVPP22_CTRL4_DP_CLK_SEL;
4472 writel(val, port->base + MVPP22_GMAC_CTRL_4_REG);
4473
4474 val = readl(port->base + MVPP2_GMAC_CTRL_2_REG);
4475 val &= ~MVPP2_GMAC_DISABLE_PADDING;
4476 writel(val, port->base + MVPP2_GMAC_CTRL_2_REG);
4477 }
4478
4479 /* The port is connected to a copper PHY */
4480 val = readl(port->base + MVPP2_GMAC_CTRL_0_REG);
4481 val &= ~MVPP2_GMAC_PORT_TYPE_MASK;
4482 writel(val, port->base + MVPP2_GMAC_CTRL_0_REG);
4483
4484 val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
4485 val |= MVPP2_GMAC_IN_BAND_AUTONEG_BYPASS |
4486 MVPP2_GMAC_AN_SPEED_EN | MVPP2_GMAC_FLOW_CTRL_AUTONEG |
4487 MVPP2_GMAC_AN_DUPLEX_EN;
4488 if (port->phy_interface == PHY_INTERFACE_MODE_SGMII)
4489 val |= MVPP2_GMAC_IN_BAND_AUTONEG;
4490 writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
4491}
4492
4493static void mvpp2_port_mii_gmac_configure(struct mvpp2_port *port)
4494{
4495 u32 val;
4496
4497 /* Force link down */
4498 val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
4499 val &= ~MVPP2_GMAC_FORCE_LINK_PASS;
4500 val |= MVPP2_GMAC_FORCE_LINK_DOWN;
4501 writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
4502
4503 /* Set the GMAC in a reset state */
4504 val = readl(port->base + MVPP2_GMAC_CTRL_2_REG);
4505 val |= MVPP2_GMAC_PORT_RESET_MASK;
4506 writel(val, port->base + MVPP2_GMAC_CTRL_2_REG);
4507
4508 /* Configure the PCS and in-band AN */
4509 val = readl(port->base + MVPP2_GMAC_CTRL_2_REG);
4510 if (port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
4511 val |= MVPP2_GMAC_INBAND_AN_MASK | MVPP2_GMAC_PCS_ENABLE_MASK;
Antoine Tenart1df22702017-09-01 11:04:52 +02004512 } else if (phy_interface_mode_is_rgmii(port->phy_interface)) {
Antoine Ténart39193572017-08-22 19:08:24 +02004513 val &= ~MVPP2_GMAC_PCS_ENABLE_MASK;
4514 val |= MVPP2_GMAC_PORT_RGMII_MASK;
4515 }
4516 writel(val, port->base + MVPP2_GMAC_CTRL_2_REG);
4517
4518 mvpp2_port_mii_gmac_configure_mode(port);
4519
4520 /* Unset the GMAC reset state */
4521 val = readl(port->base + MVPP2_GMAC_CTRL_2_REG);
4522 val &= ~MVPP2_GMAC_PORT_RESET_MASK;
4523 writel(val, port->base + MVPP2_GMAC_CTRL_2_REG);
4524
4525 /* Stop forcing link down */
4526 val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
4527 val &= ~MVPP2_GMAC_FORCE_LINK_DOWN;
4528 writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
4529}
4530
Antoine Ténart77321952017-08-22 19:08:25 +02004531static void mvpp2_port_mii_xlg_configure(struct mvpp2_port *port)
4532{
4533 u32 val;
4534
4535 if (port->gop_id != 0)
4536 return;
4537
4538 val = readl(port->base + MVPP22_XLG_CTRL0_REG);
4539 val |= MVPP22_XLG_CTRL0_RX_FLOW_CTRL_EN;
4540 writel(val, port->base + MVPP22_XLG_CTRL0_REG);
4541
4542 val = readl(port->base + MVPP22_XLG_CTRL4_REG);
4543 val &= ~MVPP22_XLG_CTRL4_MACMODSELECT_GMAC;
4544 val |= MVPP22_XLG_CTRL4_FWD_FC | MVPP22_XLG_CTRL4_FWD_PFC;
4545 writel(val, port->base + MVPP22_XLG_CTRL4_REG);
4546}
4547
Thomas Petazzoni26975822017-03-07 16:53:14 +01004548static void mvpp22_port_mii_set(struct mvpp2_port *port)
4549{
4550 u32 val;
4551
Thomas Petazzoni26975822017-03-07 16:53:14 +01004552 /* Only GOP port 0 has an XLG MAC */
4553 if (port->gop_id == 0) {
4554 val = readl(port->base + MVPP22_XLG_CTRL3_REG);
4555 val &= ~MVPP22_XLG_CTRL3_MACMODESELECT_MASK;
Antoine Ténart725757a2017-06-12 16:01:39 +02004556
4557 if (port->phy_interface == PHY_INTERFACE_MODE_XAUI ||
4558 port->phy_interface == PHY_INTERFACE_MODE_10GKR)
4559 val |= MVPP22_XLG_CTRL3_MACMODESELECT_10G;
4560 else
4561 val |= MVPP22_XLG_CTRL3_MACMODESELECT_GMAC;
4562
Thomas Petazzoni26975822017-03-07 16:53:14 +01004563 writel(val, port->base + MVPP22_XLG_CTRL3_REG);
4564 }
Thomas Petazzoni26975822017-03-07 16:53:14 +01004565}
4566
Marcin Wojtas3f518502014-07-10 16:52:13 -03004567static void mvpp2_port_mii_set(struct mvpp2_port *port)
4568{
Thomas Petazzoni26975822017-03-07 16:53:14 +01004569 if (port->priv->hw_version == MVPP22)
4570 mvpp22_port_mii_set(port);
4571
Antoine Tenart1df22702017-09-01 11:04:52 +02004572 if (phy_interface_mode_is_rgmii(port->phy_interface) ||
Antoine Ténart39193572017-08-22 19:08:24 +02004573 port->phy_interface == PHY_INTERFACE_MODE_SGMII)
4574 mvpp2_port_mii_gmac_configure(port);
Antoine Ténart77321952017-08-22 19:08:25 +02004575 else if (port->phy_interface == PHY_INTERFACE_MODE_10GKR)
4576 mvpp2_port_mii_xlg_configure(port);
Marcin Wojtas08a23752014-07-21 13:48:12 -03004577}
4578
4579static void mvpp2_port_fc_adv_enable(struct mvpp2_port *port)
4580{
4581 u32 val;
4582
4583 val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
4584 val |= MVPP2_GMAC_FC_ADV_EN;
4585 writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
Marcin Wojtas3f518502014-07-10 16:52:13 -03004586}
4587
4588static void mvpp2_port_enable(struct mvpp2_port *port)
4589{
4590 u32 val;
4591
Antoine Ténart725757a2017-06-12 16:01:39 +02004592 /* Only GOP port 0 has an XLG MAC */
4593 if (port->gop_id == 0 &&
4594 (port->phy_interface == PHY_INTERFACE_MODE_XAUI ||
4595 port->phy_interface == PHY_INTERFACE_MODE_10GKR)) {
4596 val = readl(port->base + MVPP22_XLG_CTRL0_REG);
4597 val |= MVPP22_XLG_CTRL0_PORT_EN |
4598 MVPP22_XLG_CTRL0_MAC_RESET_DIS;
4599 val &= ~MVPP22_XLG_CTRL0_MIB_CNT_DIS;
4600 writel(val, port->base + MVPP22_XLG_CTRL0_REG);
4601 } else {
4602 val = readl(port->base + MVPP2_GMAC_CTRL_0_REG);
4603 val |= MVPP2_GMAC_PORT_EN_MASK;
4604 val |= MVPP2_GMAC_MIB_CNTR_EN_MASK;
4605 writel(val, port->base + MVPP2_GMAC_CTRL_0_REG);
4606 }
Marcin Wojtas3f518502014-07-10 16:52:13 -03004607}
4608
4609static void mvpp2_port_disable(struct mvpp2_port *port)
4610{
4611 u32 val;
4612
Antoine Ténart725757a2017-06-12 16:01:39 +02004613 /* Only GOP port 0 has an XLG MAC */
4614 if (port->gop_id == 0 &&
4615 (port->phy_interface == PHY_INTERFACE_MODE_XAUI ||
4616 port->phy_interface == PHY_INTERFACE_MODE_10GKR)) {
4617 val = readl(port->base + MVPP22_XLG_CTRL0_REG);
4618 val &= ~(MVPP22_XLG_CTRL0_PORT_EN |
4619 MVPP22_XLG_CTRL0_MAC_RESET_DIS);
4620 writel(val, port->base + MVPP22_XLG_CTRL0_REG);
4621 } else {
4622 val = readl(port->base + MVPP2_GMAC_CTRL_0_REG);
4623 val &= ~(MVPP2_GMAC_PORT_EN_MASK);
4624 writel(val, port->base + MVPP2_GMAC_CTRL_0_REG);
4625 }
Marcin Wojtas3f518502014-07-10 16:52:13 -03004626}
4627
4628/* Set IEEE 802.3x Flow Control Xon Packet Transmission Mode */
4629static void mvpp2_port_periodic_xon_disable(struct mvpp2_port *port)
4630{
4631 u32 val;
4632
4633 val = readl(port->base + MVPP2_GMAC_CTRL_1_REG) &
4634 ~MVPP2_GMAC_PERIODIC_XON_EN_MASK;
4635 writel(val, port->base + MVPP2_GMAC_CTRL_1_REG);
4636}
4637
4638/* Configure loopback port */
4639static void mvpp2_port_loopback_set(struct mvpp2_port *port)
4640{
4641 u32 val;
4642
4643 val = readl(port->base + MVPP2_GMAC_CTRL_1_REG);
4644
4645 if (port->speed == 1000)
4646 val |= MVPP2_GMAC_GMII_LB_EN_MASK;
4647 else
4648 val &= ~MVPP2_GMAC_GMII_LB_EN_MASK;
4649
4650 if (port->phy_interface == PHY_INTERFACE_MODE_SGMII)
4651 val |= MVPP2_GMAC_PCS_LB_EN_MASK;
4652 else
4653 val &= ~MVPP2_GMAC_PCS_LB_EN_MASK;
4654
4655 writel(val, port->base + MVPP2_GMAC_CTRL_1_REG);
4656}
4657
4658static void mvpp2_port_reset(struct mvpp2_port *port)
4659{
4660 u32 val;
4661
4662 val = readl(port->base + MVPP2_GMAC_CTRL_2_REG) &
4663 ~MVPP2_GMAC_PORT_RESET_MASK;
4664 writel(val, port->base + MVPP2_GMAC_CTRL_2_REG);
4665
4666 while (readl(port->base + MVPP2_GMAC_CTRL_2_REG) &
4667 MVPP2_GMAC_PORT_RESET_MASK)
4668 continue;
4669}
4670
4671/* Change maximum receive size of the port */
4672static inline void mvpp2_gmac_max_rx_size_set(struct mvpp2_port *port)
4673{
4674 u32 val;
4675
4676 val = readl(port->base + MVPP2_GMAC_CTRL_0_REG);
4677 val &= ~MVPP2_GMAC_MAX_RX_SIZE_MASK;
4678 val |= (((port->pkt_size - MVPP2_MH_SIZE) / 2) <<
4679 MVPP2_GMAC_MAX_RX_SIZE_OFFS);
4680 writel(val, port->base + MVPP2_GMAC_CTRL_0_REG);
4681}
4682
Stefan Chulski76eb1b12017-08-22 19:08:26 +02004683/* Change maximum receive size of the port */
4684static inline void mvpp2_xlg_max_rx_size_set(struct mvpp2_port *port)
4685{
4686 u32 val;
4687
4688 val = readl(port->base + MVPP22_XLG_CTRL1_REG);
4689 val &= ~MVPP22_XLG_CTRL1_FRAMESIZELIMIT_MASK;
4690 val |= ((port->pkt_size - MVPP2_MH_SIZE) / 2) <<
Antoine Ténartec15ecd2017-08-25 15:24:46 +02004691 MVPP22_XLG_CTRL1_FRAMESIZELIMIT_OFFS;
Stefan Chulski76eb1b12017-08-22 19:08:26 +02004692 writel(val, port->base + MVPP22_XLG_CTRL1_REG);
4693}
4694
Marcin Wojtas3f518502014-07-10 16:52:13 -03004695/* Set defaults to the MVPP2 port */
4696static void mvpp2_defaults_set(struct mvpp2_port *port)
4697{
4698 int tx_port_num, val, queue, ptxq, lrxq;
4699
Thomas Petazzoni3d9017d2017-03-07 16:53:11 +01004700 if (port->priv->hw_version == MVPP21) {
4701 /* Configure port to loopback if needed */
4702 if (port->flags & MVPP2_F_LOOPBACK)
4703 mvpp2_port_loopback_set(port);
Marcin Wojtas3f518502014-07-10 16:52:13 -03004704
Thomas Petazzoni3d9017d2017-03-07 16:53:11 +01004705 /* Update TX FIFO MIN Threshold */
4706 val = readl(port->base + MVPP2_GMAC_PORT_FIFO_CFG_1_REG);
4707 val &= ~MVPP2_GMAC_TX_FIFO_MIN_TH_ALL_MASK;
4708 /* Min. TX threshold must be less than minimal packet length */
4709 val |= MVPP2_GMAC_TX_FIFO_MIN_TH_MASK(64 - 4 - 2);
4710 writel(val, port->base + MVPP2_GMAC_PORT_FIFO_CFG_1_REG);
4711 }
Marcin Wojtas3f518502014-07-10 16:52:13 -03004712
4713 /* Disable Legacy WRR, Disable EJP, Release from reset */
4714 tx_port_num = mvpp2_egress_port(port);
4715 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG,
4716 tx_port_num);
4717 mvpp2_write(port->priv, MVPP2_TXP_SCHED_CMD_1_REG, 0);
4718
4719 /* Close bandwidth for all queues */
4720 for (queue = 0; queue < MVPP2_MAX_TXQ; queue++) {
4721 ptxq = mvpp2_txq_phys(port->id, queue);
4722 mvpp2_write(port->priv,
4723 MVPP2_TXQ_SCHED_TOKEN_CNTR_REG(ptxq), 0);
4724 }
4725
4726 /* Set refill period to 1 usec, refill tokens
4727 * and bucket size to maximum
4728 */
4729 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PERIOD_REG,
4730 port->priv->tclk / USEC_PER_SEC);
4731 val = mvpp2_read(port->priv, MVPP2_TXP_SCHED_REFILL_REG);
4732 val &= ~MVPP2_TXP_REFILL_PERIOD_ALL_MASK;
4733 val |= MVPP2_TXP_REFILL_PERIOD_MASK(1);
4734 val |= MVPP2_TXP_REFILL_TOKENS_ALL_MASK;
4735 mvpp2_write(port->priv, MVPP2_TXP_SCHED_REFILL_REG, val);
4736 val = MVPP2_TXP_TOKEN_SIZE_MAX;
4737 mvpp2_write(port->priv, MVPP2_TXP_SCHED_TOKEN_SIZE_REG, val);
4738
4739 /* Set MaximumLowLatencyPacketSize value to 256 */
4740 mvpp2_write(port->priv, MVPP2_RX_CTRL_REG(port->id),
4741 MVPP2_RX_USE_PSEUDO_FOR_CSUM_MASK |
4742 MVPP2_RX_LOW_LATENCY_PKT_SIZE(256));
4743
4744 /* Enable Rx cache snoop */
Thomas Petazzoni09f83972017-08-03 10:41:57 +02004745 for (lrxq = 0; lrxq < port->nrxqs; lrxq++) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03004746 queue = port->rxqs[lrxq]->id;
4747 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(queue));
4748 val |= MVPP2_SNOOP_PKT_SIZE_MASK |
4749 MVPP2_SNOOP_BUF_HDR_MASK;
4750 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(queue), val);
4751 }
4752
4753 /* At default, mask all interrupts to all present cpus */
4754 mvpp2_interrupts_disable(port);
4755}
4756
4757/* Enable/disable receiving packets */
4758static void mvpp2_ingress_enable(struct mvpp2_port *port)
4759{
4760 u32 val;
4761 int lrxq, queue;
4762
Thomas Petazzoni09f83972017-08-03 10:41:57 +02004763 for (lrxq = 0; lrxq < port->nrxqs; lrxq++) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03004764 queue = port->rxqs[lrxq]->id;
4765 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(queue));
4766 val &= ~MVPP2_RXQ_DISABLE_MASK;
4767 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(queue), val);
4768 }
4769}
4770
4771static void mvpp2_ingress_disable(struct mvpp2_port *port)
4772{
4773 u32 val;
4774 int lrxq, queue;
4775
Thomas Petazzoni09f83972017-08-03 10:41:57 +02004776 for (lrxq = 0; lrxq < port->nrxqs; lrxq++) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03004777 queue = port->rxqs[lrxq]->id;
4778 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(queue));
4779 val |= MVPP2_RXQ_DISABLE_MASK;
4780 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(queue), val);
4781 }
4782}
4783
4784/* Enable transmit via physical egress queue
4785 * - HW starts take descriptors from DRAM
4786 */
4787static void mvpp2_egress_enable(struct mvpp2_port *port)
4788{
4789 u32 qmap;
4790 int queue;
4791 int tx_port_num = mvpp2_egress_port(port);
4792
4793 /* Enable all initialized TXs. */
4794 qmap = 0;
Thomas Petazzoni09f83972017-08-03 10:41:57 +02004795 for (queue = 0; queue < port->ntxqs; queue++) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03004796 struct mvpp2_tx_queue *txq = port->txqs[queue];
4797
Markus Elfringdbbb2f02017-04-17 14:07:52 +02004798 if (txq->descs)
Marcin Wojtas3f518502014-07-10 16:52:13 -03004799 qmap |= (1 << queue);
4800 }
4801
4802 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG, tx_port_num);
4803 mvpp2_write(port->priv, MVPP2_TXP_SCHED_Q_CMD_REG, qmap);
4804}
4805
4806/* Disable transmit via physical egress queue
4807 * - HW doesn't take descriptors from DRAM
4808 */
4809static void mvpp2_egress_disable(struct mvpp2_port *port)
4810{
4811 u32 reg_data;
4812 int delay;
4813 int tx_port_num = mvpp2_egress_port(port);
4814
4815 /* Issue stop command for active channels only */
4816 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG, tx_port_num);
4817 reg_data = (mvpp2_read(port->priv, MVPP2_TXP_SCHED_Q_CMD_REG)) &
4818 MVPP2_TXP_SCHED_ENQ_MASK;
4819 if (reg_data != 0)
4820 mvpp2_write(port->priv, MVPP2_TXP_SCHED_Q_CMD_REG,
4821 (reg_data << MVPP2_TXP_SCHED_DISQ_OFFSET));
4822
4823 /* Wait for all Tx activity to terminate. */
4824 delay = 0;
4825 do {
4826 if (delay >= MVPP2_TX_DISABLE_TIMEOUT_MSEC) {
4827 netdev_warn(port->dev,
4828 "Tx stop timed out, status=0x%08x\n",
4829 reg_data);
4830 break;
4831 }
4832 mdelay(1);
4833 delay++;
4834
4835 /* Check port TX Command register that all
4836 * Tx queues are stopped
4837 */
4838 reg_data = mvpp2_read(port->priv, MVPP2_TXP_SCHED_Q_CMD_REG);
4839 } while (reg_data & MVPP2_TXP_SCHED_ENQ_MASK);
4840}
4841
4842/* Rx descriptors helper methods */
4843
4844/* Get number of Rx descriptors occupied by received packets */
4845static inline int
4846mvpp2_rxq_received(struct mvpp2_port *port, int rxq_id)
4847{
4848 u32 val = mvpp2_read(port->priv, MVPP2_RXQ_STATUS_REG(rxq_id));
4849
4850 return val & MVPP2_RXQ_OCCUPIED_MASK;
4851}
4852
4853/* Update Rx queue status with the number of occupied and available
4854 * Rx descriptor slots.
4855 */
4856static inline void
4857mvpp2_rxq_status_update(struct mvpp2_port *port, int rxq_id,
4858 int used_count, int free_count)
4859{
4860 /* Decrement the number of used descriptors and increment count
4861 * increment the number of free descriptors.
4862 */
4863 u32 val = used_count | (free_count << MVPP2_RXQ_NUM_NEW_OFFSET);
4864
4865 mvpp2_write(port->priv, MVPP2_RXQ_STATUS_UPDATE_REG(rxq_id), val);
4866}
4867
4868/* Get pointer to next RX descriptor to be processed by SW */
4869static inline struct mvpp2_rx_desc *
4870mvpp2_rxq_next_desc_get(struct mvpp2_rx_queue *rxq)
4871{
4872 int rx_desc = rxq->next_desc_to_proc;
4873
4874 rxq->next_desc_to_proc = MVPP2_QUEUE_NEXT_DESC(rxq, rx_desc);
4875 prefetch(rxq->descs + rxq->next_desc_to_proc);
4876 return rxq->descs + rx_desc;
4877}
4878
4879/* Set rx queue offset */
4880static void mvpp2_rxq_offset_set(struct mvpp2_port *port,
4881 int prxq, int offset)
4882{
4883 u32 val;
4884
4885 /* Convert offset from bytes to units of 32 bytes */
4886 offset = offset >> 5;
4887
4888 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(prxq));
4889 val &= ~MVPP2_RXQ_PACKET_OFFSET_MASK;
4890
4891 /* Offset is in */
4892 val |= ((offset << MVPP2_RXQ_PACKET_OFFSET_OFFS) &
4893 MVPP2_RXQ_PACKET_OFFSET_MASK);
4894
4895 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(prxq), val);
4896}
4897
Marcin Wojtas3f518502014-07-10 16:52:13 -03004898/* Tx descriptors helper methods */
4899
Marcin Wojtas3f518502014-07-10 16:52:13 -03004900/* Get pointer to next Tx descriptor to be processed (send) by HW */
4901static struct mvpp2_tx_desc *
4902mvpp2_txq_next_desc_get(struct mvpp2_tx_queue *txq)
4903{
4904 int tx_desc = txq->next_desc_to_proc;
4905
4906 txq->next_desc_to_proc = MVPP2_QUEUE_NEXT_DESC(txq, tx_desc);
4907 return txq->descs + tx_desc;
4908}
4909
Thomas Petazzonie0af22d2017-06-22 14:23:18 +02004910/* Update HW with number of aggregated Tx descriptors to be sent
4911 *
4912 * Called only from mvpp2_tx(), so migration is disabled, using
4913 * smp_processor_id() is OK.
4914 */
Marcin Wojtas3f518502014-07-10 16:52:13 -03004915static void mvpp2_aggr_txq_pend_desc_add(struct mvpp2_port *port, int pending)
4916{
4917 /* aggregated access - relevant TXQ number is written in TX desc */
Thomas Petazzonia7868412017-03-07 16:53:13 +01004918 mvpp2_percpu_write(port->priv, smp_processor_id(),
4919 MVPP2_AGGR_TXQ_UPDATE_REG, pending);
Marcin Wojtas3f518502014-07-10 16:52:13 -03004920}
4921
4922
4923/* Check if there are enough free descriptors in aggregated txq.
4924 * If not, update the number of occupied descriptors and repeat the check.
Thomas Petazzonie0af22d2017-06-22 14:23:18 +02004925 *
4926 * Called only from mvpp2_tx(), so migration is disabled, using
4927 * smp_processor_id() is OK.
Marcin Wojtas3f518502014-07-10 16:52:13 -03004928 */
4929static int mvpp2_aggr_desc_num_check(struct mvpp2 *priv,
4930 struct mvpp2_tx_queue *aggr_txq, int num)
4931{
4932 if ((aggr_txq->count + num) > aggr_txq->size) {
4933 /* Update number of occupied aggregated Tx descriptors */
4934 int cpu = smp_processor_id();
4935 u32 val = mvpp2_read(priv, MVPP2_AGGR_TXQ_STATUS_REG(cpu));
4936
4937 aggr_txq->count = val & MVPP2_AGGR_TXQ_PENDING_MASK;
4938 }
4939
4940 if ((aggr_txq->count + num) > aggr_txq->size)
4941 return -ENOMEM;
4942
4943 return 0;
4944}
4945
Thomas Petazzonie0af22d2017-06-22 14:23:18 +02004946/* Reserved Tx descriptors allocation request
4947 *
4948 * Called only from mvpp2_txq_reserved_desc_num_proc(), itself called
4949 * only by mvpp2_tx(), so migration is disabled, using
4950 * smp_processor_id() is OK.
4951 */
Marcin Wojtas3f518502014-07-10 16:52:13 -03004952static int mvpp2_txq_alloc_reserved_desc(struct mvpp2 *priv,
4953 struct mvpp2_tx_queue *txq, int num)
4954{
4955 u32 val;
Thomas Petazzonia7868412017-03-07 16:53:13 +01004956 int cpu = smp_processor_id();
Marcin Wojtas3f518502014-07-10 16:52:13 -03004957
4958 val = (txq->id << MVPP2_TXQ_RSVD_REQ_Q_OFFSET) | num;
Thomas Petazzonia7868412017-03-07 16:53:13 +01004959 mvpp2_percpu_write(priv, cpu, MVPP2_TXQ_RSVD_REQ_REG, val);
Marcin Wojtas3f518502014-07-10 16:52:13 -03004960
Thomas Petazzonia7868412017-03-07 16:53:13 +01004961 val = mvpp2_percpu_read(priv, cpu, MVPP2_TXQ_RSVD_RSLT_REG);
Marcin Wojtas3f518502014-07-10 16:52:13 -03004962
4963 return val & MVPP2_TXQ_RSVD_RSLT_MASK;
4964}
4965
4966/* Check if there are enough reserved descriptors for transmission.
4967 * If not, request chunk of reserved descriptors and check again.
4968 */
4969static int mvpp2_txq_reserved_desc_num_proc(struct mvpp2 *priv,
4970 struct mvpp2_tx_queue *txq,
4971 struct mvpp2_txq_pcpu *txq_pcpu,
4972 int num)
4973{
4974 int req, cpu, desc_count;
4975
4976 if (txq_pcpu->reserved_num >= num)
4977 return 0;
4978
4979 /* Not enough descriptors reserved! Update the reserved descriptor
4980 * count and check again.
4981 */
4982
4983 desc_count = 0;
4984 /* Compute total of used descriptors */
4985 for_each_present_cpu(cpu) {
4986 struct mvpp2_txq_pcpu *txq_pcpu_aux;
4987
4988 txq_pcpu_aux = per_cpu_ptr(txq->pcpu, cpu);
4989 desc_count += txq_pcpu_aux->count;
4990 desc_count += txq_pcpu_aux->reserved_num;
4991 }
4992
4993 req = max(MVPP2_CPU_DESC_CHUNK, num - txq_pcpu->reserved_num);
4994 desc_count += req;
4995
4996 if (desc_count >
4997 (txq->size - (num_present_cpus() * MVPP2_CPU_DESC_CHUNK)))
4998 return -ENOMEM;
4999
5000 txq_pcpu->reserved_num += mvpp2_txq_alloc_reserved_desc(priv, txq, req);
5001
5002 /* OK, the descriptor cound has been updated: check again. */
5003 if (txq_pcpu->reserved_num < num)
5004 return -ENOMEM;
5005 return 0;
5006}
5007
5008/* Release the last allocated Tx descriptor. Useful to handle DMA
5009 * mapping failures in the Tx path.
5010 */
5011static void mvpp2_txq_desc_put(struct mvpp2_tx_queue *txq)
5012{
5013 if (txq->next_desc_to_proc == 0)
5014 txq->next_desc_to_proc = txq->last_desc - 1;
5015 else
5016 txq->next_desc_to_proc--;
5017}
5018
5019/* Set Tx descriptors fields relevant for CSUM calculation */
5020static u32 mvpp2_txq_desc_csum(int l3_offs, int l3_proto,
5021 int ip_hdr_len, int l4_proto)
5022{
5023 u32 command;
5024
5025 /* fields: L3_offset, IP_hdrlen, L3_type, G_IPv4_chk,
5026 * G_L4_chk, L4_type required only for checksum calculation
5027 */
5028 command = (l3_offs << MVPP2_TXD_L3_OFF_SHIFT);
5029 command |= (ip_hdr_len << MVPP2_TXD_IP_HLEN_SHIFT);
5030 command |= MVPP2_TXD_IP_CSUM_DISABLE;
5031
5032 if (l3_proto == swab16(ETH_P_IP)) {
5033 command &= ~MVPP2_TXD_IP_CSUM_DISABLE; /* enable IPv4 csum */
5034 command &= ~MVPP2_TXD_L3_IP6; /* enable IPv4 */
5035 } else {
5036 command |= MVPP2_TXD_L3_IP6; /* enable IPv6 */
5037 }
5038
5039 if (l4_proto == IPPROTO_TCP) {
5040 command &= ~MVPP2_TXD_L4_UDP; /* enable TCP */
5041 command &= ~MVPP2_TXD_L4_CSUM_FRAG; /* generate L4 csum */
5042 } else if (l4_proto == IPPROTO_UDP) {
5043 command |= MVPP2_TXD_L4_UDP; /* enable UDP */
5044 command &= ~MVPP2_TXD_L4_CSUM_FRAG; /* generate L4 csum */
5045 } else {
5046 command |= MVPP2_TXD_L4_CSUM_NOT;
5047 }
5048
5049 return command;
5050}
5051
5052/* Get number of sent descriptors and decrement counter.
5053 * The number of sent descriptors is returned.
5054 * Per-CPU access
Thomas Petazzonie0af22d2017-06-22 14:23:18 +02005055 *
5056 * Called only from mvpp2_txq_done(), called from mvpp2_tx()
5057 * (migration disabled) and from the TX completion tasklet (migration
5058 * disabled) so using smp_processor_id() is OK.
Marcin Wojtas3f518502014-07-10 16:52:13 -03005059 */
5060static inline int mvpp2_txq_sent_desc_proc(struct mvpp2_port *port,
5061 struct mvpp2_tx_queue *txq)
5062{
5063 u32 val;
5064
5065 /* Reading status reg resets transmitted descriptor counter */
Thomas Petazzonia7868412017-03-07 16:53:13 +01005066 val = mvpp2_percpu_read(port->priv, smp_processor_id(),
5067 MVPP2_TXQ_SENT_REG(txq->id));
Marcin Wojtas3f518502014-07-10 16:52:13 -03005068
5069 return (val & MVPP2_TRANSMITTED_COUNT_MASK) >>
5070 MVPP2_TRANSMITTED_COUNT_OFFSET;
5071}
5072
Thomas Petazzonie0af22d2017-06-22 14:23:18 +02005073/* Called through on_each_cpu(), so runs on all CPUs, with migration
5074 * disabled, therefore using smp_processor_id() is OK.
5075 */
Marcin Wojtas3f518502014-07-10 16:52:13 -03005076static void mvpp2_txq_sent_counter_clear(void *arg)
5077{
5078 struct mvpp2_port *port = arg;
5079 int queue;
5080
Thomas Petazzoni09f83972017-08-03 10:41:57 +02005081 for (queue = 0; queue < port->ntxqs; queue++) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03005082 int id = port->txqs[queue]->id;
5083
Thomas Petazzonia7868412017-03-07 16:53:13 +01005084 mvpp2_percpu_read(port->priv, smp_processor_id(),
5085 MVPP2_TXQ_SENT_REG(id));
Marcin Wojtas3f518502014-07-10 16:52:13 -03005086 }
5087}
5088
5089/* Set max sizes for Tx queues */
5090static void mvpp2_txp_max_tx_size_set(struct mvpp2_port *port)
5091{
5092 u32 val, size, mtu;
5093 int txq, tx_port_num;
5094
5095 mtu = port->pkt_size * 8;
5096 if (mtu > MVPP2_TXP_MTU_MAX)
5097 mtu = MVPP2_TXP_MTU_MAX;
5098
5099 /* WA for wrong Token bucket update: Set MTU value = 3*real MTU value */
5100 mtu = 3 * mtu;
5101
5102 /* Indirect access to registers */
5103 tx_port_num = mvpp2_egress_port(port);
5104 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG, tx_port_num);
5105
5106 /* Set MTU */
5107 val = mvpp2_read(port->priv, MVPP2_TXP_SCHED_MTU_REG);
5108 val &= ~MVPP2_TXP_MTU_MAX;
5109 val |= mtu;
5110 mvpp2_write(port->priv, MVPP2_TXP_SCHED_MTU_REG, val);
5111
5112 /* TXP token size and all TXQs token size must be larger that MTU */
5113 val = mvpp2_read(port->priv, MVPP2_TXP_SCHED_TOKEN_SIZE_REG);
5114 size = val & MVPP2_TXP_TOKEN_SIZE_MAX;
5115 if (size < mtu) {
5116 size = mtu;
5117 val &= ~MVPP2_TXP_TOKEN_SIZE_MAX;
5118 val |= size;
5119 mvpp2_write(port->priv, MVPP2_TXP_SCHED_TOKEN_SIZE_REG, val);
5120 }
5121
Thomas Petazzoni09f83972017-08-03 10:41:57 +02005122 for (txq = 0; txq < port->ntxqs; txq++) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03005123 val = mvpp2_read(port->priv,
5124 MVPP2_TXQ_SCHED_TOKEN_SIZE_REG(txq));
5125 size = val & MVPP2_TXQ_TOKEN_SIZE_MAX;
5126
5127 if (size < mtu) {
5128 size = mtu;
5129 val &= ~MVPP2_TXQ_TOKEN_SIZE_MAX;
5130 val |= size;
5131 mvpp2_write(port->priv,
5132 MVPP2_TXQ_SCHED_TOKEN_SIZE_REG(txq),
5133 val);
5134 }
5135 }
5136}
5137
5138/* Set the number of packets that will be received before Rx interrupt
5139 * will be generated by HW.
5140 */
5141static void mvpp2_rx_pkts_coal_set(struct mvpp2_port *port,
Thomas Petazzonid63f9e42017-02-21 11:28:02 +01005142 struct mvpp2_rx_queue *rxq)
Marcin Wojtas3f518502014-07-10 16:52:13 -03005143{
Thomas Petazzonia704bb52017-06-10 23:18:22 +02005144 int cpu = get_cpu();
Thomas Petazzonia7868412017-03-07 16:53:13 +01005145
Thomas Petazzonif8b0d5f2017-02-21 11:28:03 +01005146 if (rxq->pkts_coal > MVPP2_OCCUPIED_THRESH_MASK)
5147 rxq->pkts_coal = MVPP2_OCCUPIED_THRESH_MASK;
Marcin Wojtas3f518502014-07-10 16:52:13 -03005148
Thomas Petazzonia7868412017-03-07 16:53:13 +01005149 mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_NUM_REG, rxq->id);
5150 mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_THRESH_REG,
5151 rxq->pkts_coal);
Thomas Petazzonia704bb52017-06-10 23:18:22 +02005152
5153 put_cpu();
Marcin Wojtas3f518502014-07-10 16:52:13 -03005154}
5155
Thomas Petazzoni213f4282017-08-03 10:42:00 +02005156/* For some reason in the LSP this is done on each CPU. Why ? */
5157static void mvpp2_tx_pkts_coal_set(struct mvpp2_port *port,
5158 struct mvpp2_tx_queue *txq)
5159{
5160 int cpu = get_cpu();
5161 u32 val;
5162
5163 if (txq->done_pkts_coal > MVPP2_TXQ_THRESH_MASK)
5164 txq->done_pkts_coal = MVPP2_TXQ_THRESH_MASK;
5165
5166 val = (txq->done_pkts_coal << MVPP2_TXQ_THRESH_OFFSET);
5167 mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_NUM_REG, txq->id);
5168 mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_THRESH_REG, val);
5169
5170 put_cpu();
5171}
5172
Thomas Petazzoniab426762017-02-21 11:28:04 +01005173static u32 mvpp2_usec_to_cycles(u32 usec, unsigned long clk_hz)
5174{
5175 u64 tmp = (u64)clk_hz * usec;
5176
5177 do_div(tmp, USEC_PER_SEC);
5178
5179 return tmp > U32_MAX ? U32_MAX : tmp;
5180}
5181
5182static u32 mvpp2_cycles_to_usec(u32 cycles, unsigned long clk_hz)
5183{
5184 u64 tmp = (u64)cycles * USEC_PER_SEC;
5185
5186 do_div(tmp, clk_hz);
5187
5188 return tmp > U32_MAX ? U32_MAX : tmp;
5189}
5190
Marcin Wojtas3f518502014-07-10 16:52:13 -03005191/* Set the time delay in usec before Rx interrupt */
5192static void mvpp2_rx_time_coal_set(struct mvpp2_port *port,
Thomas Petazzonid63f9e42017-02-21 11:28:02 +01005193 struct mvpp2_rx_queue *rxq)
Marcin Wojtas3f518502014-07-10 16:52:13 -03005194{
Thomas Petazzoniab426762017-02-21 11:28:04 +01005195 unsigned long freq = port->priv->tclk;
5196 u32 val = mvpp2_usec_to_cycles(rxq->time_coal, freq);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005197
Thomas Petazzoniab426762017-02-21 11:28:04 +01005198 if (val > MVPP2_MAX_ISR_RX_THRESHOLD) {
5199 rxq->time_coal =
5200 mvpp2_cycles_to_usec(MVPP2_MAX_ISR_RX_THRESHOLD, freq);
5201
5202 /* re-evaluate to get actual register value */
5203 val = mvpp2_usec_to_cycles(rxq->time_coal, freq);
5204 }
5205
Marcin Wojtas3f518502014-07-10 16:52:13 -03005206 mvpp2_write(port->priv, MVPP2_ISR_RX_THRESHOLD_REG(rxq->id), val);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005207}
5208
Thomas Petazzoni213f4282017-08-03 10:42:00 +02005209static void mvpp2_tx_time_coal_set(struct mvpp2_port *port)
5210{
5211 unsigned long freq = port->priv->tclk;
5212 u32 val = mvpp2_usec_to_cycles(port->tx_time_coal, freq);
5213
5214 if (val > MVPP2_MAX_ISR_TX_THRESHOLD) {
5215 port->tx_time_coal =
5216 mvpp2_cycles_to_usec(MVPP2_MAX_ISR_TX_THRESHOLD, freq);
5217
5218 /* re-evaluate to get actual register value */
5219 val = mvpp2_usec_to_cycles(port->tx_time_coal, freq);
5220 }
5221
5222 mvpp2_write(port->priv, MVPP2_ISR_TX_THRESHOLD_REG(port->id), val);
5223}
5224
Marcin Wojtas3f518502014-07-10 16:52:13 -03005225/* Free Tx queue skbuffs */
5226static void mvpp2_txq_bufs_free(struct mvpp2_port *port,
5227 struct mvpp2_tx_queue *txq,
5228 struct mvpp2_txq_pcpu *txq_pcpu, int num)
5229{
5230 int i;
5231
5232 for (i = 0; i < num; i++) {
Thomas Petazzoni83544912016-12-21 11:28:49 +01005233 struct mvpp2_txq_pcpu_buf *tx_buf =
5234 txq_pcpu->buffs + txq_pcpu->txq_get_index;
Marcin Wojtas3f518502014-07-10 16:52:13 -03005235
Thomas Petazzoni20396132017-03-07 16:53:00 +01005236 dma_unmap_single(port->dev->dev.parent, tx_buf->dma,
Thomas Petazzoni83544912016-12-21 11:28:49 +01005237 tx_buf->size, DMA_TO_DEVICE);
Thomas Petazzoni36fb7432017-02-21 11:28:05 +01005238 if (tx_buf->skb)
5239 dev_kfree_skb_any(tx_buf->skb);
5240
5241 mvpp2_txq_inc_get(txq_pcpu);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005242 }
5243}
5244
5245static inline struct mvpp2_rx_queue *mvpp2_get_rx_queue(struct mvpp2_port *port,
5246 u32 cause)
5247{
5248 int queue = fls(cause) - 1;
5249
5250 return port->rxqs[queue];
5251}
5252
5253static inline struct mvpp2_tx_queue *mvpp2_get_tx_queue(struct mvpp2_port *port,
5254 u32 cause)
5255{
Marcin Wojtasedc660f2015-08-06 19:00:30 +02005256 int queue = fls(cause) - 1;
Marcin Wojtas3f518502014-07-10 16:52:13 -03005257
5258 return port->txqs[queue];
5259}
5260
5261/* Handle end of transmission */
5262static void mvpp2_txq_done(struct mvpp2_port *port, struct mvpp2_tx_queue *txq,
5263 struct mvpp2_txq_pcpu *txq_pcpu)
5264{
5265 struct netdev_queue *nq = netdev_get_tx_queue(port->dev, txq->log_id);
5266 int tx_done;
5267
5268 if (txq_pcpu->cpu != smp_processor_id())
5269 netdev_err(port->dev, "wrong cpu on the end of Tx processing\n");
5270
5271 tx_done = mvpp2_txq_sent_desc_proc(port, txq);
5272 if (!tx_done)
5273 return;
5274 mvpp2_txq_bufs_free(port, txq, txq_pcpu, tx_done);
5275
5276 txq_pcpu->count -= tx_done;
5277
5278 if (netif_tx_queue_stopped(nq))
5279 if (txq_pcpu->size - txq_pcpu->count >= MAX_SKB_FRAGS + 1)
5280 netif_tx_wake_queue(nq);
5281}
5282
Thomas Petazzoni213f4282017-08-03 10:42:00 +02005283static unsigned int mvpp2_tx_done(struct mvpp2_port *port, u32 cause,
5284 int cpu)
Marcin Wojtasedc660f2015-08-06 19:00:30 +02005285{
5286 struct mvpp2_tx_queue *txq;
5287 struct mvpp2_txq_pcpu *txq_pcpu;
5288 unsigned int tx_todo = 0;
5289
5290 while (cause) {
5291 txq = mvpp2_get_tx_queue(port, cause);
5292 if (!txq)
5293 break;
5294
Thomas Petazzoni213f4282017-08-03 10:42:00 +02005295 txq_pcpu = per_cpu_ptr(txq->pcpu, cpu);
Marcin Wojtasedc660f2015-08-06 19:00:30 +02005296
5297 if (txq_pcpu->count) {
5298 mvpp2_txq_done(port, txq, txq_pcpu);
5299 tx_todo += txq_pcpu->count;
5300 }
5301
5302 cause &= ~(1 << txq->log_id);
5303 }
5304 return tx_todo;
5305}
5306
Marcin Wojtas3f518502014-07-10 16:52:13 -03005307/* Rx/Tx queue initialization/cleanup methods */
5308
5309/* Allocate and initialize descriptors for aggr TXQ */
5310static int mvpp2_aggr_txq_init(struct platform_device *pdev,
Antoine Ténart85affd72017-08-23 09:46:55 +02005311 struct mvpp2_tx_queue *aggr_txq, int cpu,
Marcin Wojtas3f518502014-07-10 16:52:13 -03005312 struct mvpp2 *priv)
5313{
Thomas Petazzonib02f31f2017-03-07 16:53:12 +01005314 u32 txq_dma;
5315
Marcin Wojtas3f518502014-07-10 16:52:13 -03005316 /* Allocate memory for TX descriptors */
5317 aggr_txq->descs = dma_alloc_coherent(&pdev->dev,
Antoine Ténart85affd72017-08-23 09:46:55 +02005318 MVPP2_AGGR_TXQ_SIZE * MVPP2_DESC_ALIGNED_SIZE,
Thomas Petazzoni20396132017-03-07 16:53:00 +01005319 &aggr_txq->descs_dma, GFP_KERNEL);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005320 if (!aggr_txq->descs)
5321 return -ENOMEM;
5322
Marcin Wojtas3f518502014-07-10 16:52:13 -03005323 aggr_txq->last_desc = aggr_txq->size - 1;
5324
5325 /* Aggr TXQ no reset WA */
5326 aggr_txq->next_desc_to_proc = mvpp2_read(priv,
5327 MVPP2_AGGR_TXQ_INDEX_REG(cpu));
5328
Thomas Petazzonib02f31f2017-03-07 16:53:12 +01005329 /* Set Tx descriptors queue starting address indirect
5330 * access
5331 */
5332 if (priv->hw_version == MVPP21)
5333 txq_dma = aggr_txq->descs_dma;
5334 else
5335 txq_dma = aggr_txq->descs_dma >>
5336 MVPP22_AGGR_TXQ_DESC_ADDR_OFFS;
5337
5338 mvpp2_write(priv, MVPP2_AGGR_TXQ_DESC_ADDR_REG(cpu), txq_dma);
Antoine Ténart85affd72017-08-23 09:46:55 +02005339 mvpp2_write(priv, MVPP2_AGGR_TXQ_DESC_SIZE_REG(cpu),
5340 MVPP2_AGGR_TXQ_SIZE);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005341
5342 return 0;
5343}
5344
5345/* Create a specified Rx queue */
5346static int mvpp2_rxq_init(struct mvpp2_port *port,
5347 struct mvpp2_rx_queue *rxq)
5348
5349{
Thomas Petazzonib02f31f2017-03-07 16:53:12 +01005350 u32 rxq_dma;
Thomas Petazzonia7868412017-03-07 16:53:13 +01005351 int cpu;
Thomas Petazzonib02f31f2017-03-07 16:53:12 +01005352
Marcin Wojtas3f518502014-07-10 16:52:13 -03005353 rxq->size = port->rx_ring_size;
5354
5355 /* Allocate memory for RX descriptors */
5356 rxq->descs = dma_alloc_coherent(port->dev->dev.parent,
5357 rxq->size * MVPP2_DESC_ALIGNED_SIZE,
Thomas Petazzoni20396132017-03-07 16:53:00 +01005358 &rxq->descs_dma, GFP_KERNEL);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005359 if (!rxq->descs)
5360 return -ENOMEM;
5361
Marcin Wojtas3f518502014-07-10 16:52:13 -03005362 rxq->last_desc = rxq->size - 1;
5363
5364 /* Zero occupied and non-occupied counters - direct access */
5365 mvpp2_write(port->priv, MVPP2_RXQ_STATUS_REG(rxq->id), 0);
5366
5367 /* Set Rx descriptors queue starting address - indirect access */
Thomas Petazzonia704bb52017-06-10 23:18:22 +02005368 cpu = get_cpu();
Thomas Petazzonia7868412017-03-07 16:53:13 +01005369 mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_NUM_REG, rxq->id);
Thomas Petazzonib02f31f2017-03-07 16:53:12 +01005370 if (port->priv->hw_version == MVPP21)
5371 rxq_dma = rxq->descs_dma;
5372 else
5373 rxq_dma = rxq->descs_dma >> MVPP22_DESC_ADDR_OFFS;
Thomas Petazzonia7868412017-03-07 16:53:13 +01005374 mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_DESC_ADDR_REG, rxq_dma);
5375 mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_DESC_SIZE_REG, rxq->size);
5376 mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_INDEX_REG, 0);
Thomas Petazzonia704bb52017-06-10 23:18:22 +02005377 put_cpu();
Marcin Wojtas3f518502014-07-10 16:52:13 -03005378
5379 /* Set Offset */
5380 mvpp2_rxq_offset_set(port, rxq->id, NET_SKB_PAD);
5381
5382 /* Set coalescing pkts and time */
Thomas Petazzonid63f9e42017-02-21 11:28:02 +01005383 mvpp2_rx_pkts_coal_set(port, rxq);
5384 mvpp2_rx_time_coal_set(port, rxq);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005385
5386 /* Add number of descriptors ready for receiving packets */
5387 mvpp2_rxq_status_update(port, rxq->id, 0, rxq->size);
5388
5389 return 0;
5390}
5391
5392/* Push packets received by the RXQ to BM pool */
5393static void mvpp2_rxq_drop_pkts(struct mvpp2_port *port,
5394 struct mvpp2_rx_queue *rxq)
5395{
5396 int rx_received, i;
5397
5398 rx_received = mvpp2_rxq_received(port, rxq->id);
5399 if (!rx_received)
5400 return;
5401
5402 for (i = 0; i < rx_received; i++) {
5403 struct mvpp2_rx_desc *rx_desc = mvpp2_rxq_next_desc_get(rxq);
Thomas Petazzoni56b8aae2017-06-10 23:18:21 +02005404 u32 status = mvpp2_rxdesc_status_get(port, rx_desc);
5405 int pool;
Marcin Wojtas3f518502014-07-10 16:52:13 -03005406
Thomas Petazzoni56b8aae2017-06-10 23:18:21 +02005407 pool = (status & MVPP2_RXD_BM_POOL_ID_MASK) >>
5408 MVPP2_RXD_BM_POOL_ID_OFFS;
5409
Thomas Petazzoni7d7627b2017-06-22 14:23:20 +02005410 mvpp2_bm_pool_put(port, pool,
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01005411 mvpp2_rxdesc_dma_addr_get(port, rx_desc),
5412 mvpp2_rxdesc_cookie_get(port, rx_desc));
Marcin Wojtas3f518502014-07-10 16:52:13 -03005413 }
5414 mvpp2_rxq_status_update(port, rxq->id, rx_received, rx_received);
5415}
5416
5417/* Cleanup Rx queue */
5418static void mvpp2_rxq_deinit(struct mvpp2_port *port,
5419 struct mvpp2_rx_queue *rxq)
5420{
Thomas Petazzonia7868412017-03-07 16:53:13 +01005421 int cpu;
5422
Marcin Wojtas3f518502014-07-10 16:52:13 -03005423 mvpp2_rxq_drop_pkts(port, rxq);
5424
5425 if (rxq->descs)
5426 dma_free_coherent(port->dev->dev.parent,
5427 rxq->size * MVPP2_DESC_ALIGNED_SIZE,
5428 rxq->descs,
Thomas Petazzoni20396132017-03-07 16:53:00 +01005429 rxq->descs_dma);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005430
5431 rxq->descs = NULL;
5432 rxq->last_desc = 0;
5433 rxq->next_desc_to_proc = 0;
Thomas Petazzoni20396132017-03-07 16:53:00 +01005434 rxq->descs_dma = 0;
Marcin Wojtas3f518502014-07-10 16:52:13 -03005435
5436 /* Clear Rx descriptors queue starting address and size;
5437 * free descriptor number
5438 */
5439 mvpp2_write(port->priv, MVPP2_RXQ_STATUS_REG(rxq->id), 0);
Thomas Petazzonia704bb52017-06-10 23:18:22 +02005440 cpu = get_cpu();
Thomas Petazzonia7868412017-03-07 16:53:13 +01005441 mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_NUM_REG, rxq->id);
5442 mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_DESC_ADDR_REG, 0);
5443 mvpp2_percpu_write(port->priv, cpu, MVPP2_RXQ_DESC_SIZE_REG, 0);
Thomas Petazzonia704bb52017-06-10 23:18:22 +02005444 put_cpu();
Marcin Wojtas3f518502014-07-10 16:52:13 -03005445}
5446
5447/* Create and initialize a Tx queue */
5448static int mvpp2_txq_init(struct mvpp2_port *port,
5449 struct mvpp2_tx_queue *txq)
5450{
5451 u32 val;
5452 int cpu, desc, desc_per_txq, tx_port_num;
5453 struct mvpp2_txq_pcpu *txq_pcpu;
5454
5455 txq->size = port->tx_ring_size;
5456
5457 /* Allocate memory for Tx descriptors */
5458 txq->descs = dma_alloc_coherent(port->dev->dev.parent,
5459 txq->size * MVPP2_DESC_ALIGNED_SIZE,
Thomas Petazzoni20396132017-03-07 16:53:00 +01005460 &txq->descs_dma, GFP_KERNEL);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005461 if (!txq->descs)
5462 return -ENOMEM;
5463
Marcin Wojtas3f518502014-07-10 16:52:13 -03005464 txq->last_desc = txq->size - 1;
5465
5466 /* Set Tx descriptors queue starting address - indirect access */
Thomas Petazzonia704bb52017-06-10 23:18:22 +02005467 cpu = get_cpu();
Thomas Petazzonia7868412017-03-07 16:53:13 +01005468 mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_NUM_REG, txq->id);
5469 mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_DESC_ADDR_REG,
5470 txq->descs_dma);
5471 mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_DESC_SIZE_REG,
5472 txq->size & MVPP2_TXQ_DESC_SIZE_MASK);
5473 mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_INDEX_REG, 0);
5474 mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_RSVD_CLR_REG,
5475 txq->id << MVPP2_TXQ_RSVD_CLR_OFFSET);
5476 val = mvpp2_percpu_read(port->priv, cpu, MVPP2_TXQ_PENDING_REG);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005477 val &= ~MVPP2_TXQ_PENDING_MASK;
Thomas Petazzonia7868412017-03-07 16:53:13 +01005478 mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_PENDING_REG, val);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005479
5480 /* Calculate base address in prefetch buffer. We reserve 16 descriptors
5481 * for each existing TXQ.
5482 * TCONTS for PON port must be continuous from 0 to MVPP2_MAX_TCONT
5483 * GBE ports assumed to be continious from 0 to MVPP2_MAX_PORTS
5484 */
5485 desc_per_txq = 16;
5486 desc = (port->id * MVPP2_MAX_TXQ * desc_per_txq) +
5487 (txq->log_id * desc_per_txq);
5488
Thomas Petazzonia7868412017-03-07 16:53:13 +01005489 mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_PREF_BUF_REG,
5490 MVPP2_PREF_BUF_PTR(desc) | MVPP2_PREF_BUF_SIZE_16 |
5491 MVPP2_PREF_BUF_THRESH(desc_per_txq / 2));
Thomas Petazzonia704bb52017-06-10 23:18:22 +02005492 put_cpu();
Marcin Wojtas3f518502014-07-10 16:52:13 -03005493
5494 /* WRR / EJP configuration - indirect access */
5495 tx_port_num = mvpp2_egress_port(port);
5496 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG, tx_port_num);
5497
5498 val = mvpp2_read(port->priv, MVPP2_TXQ_SCHED_REFILL_REG(txq->log_id));
5499 val &= ~MVPP2_TXQ_REFILL_PERIOD_ALL_MASK;
5500 val |= MVPP2_TXQ_REFILL_PERIOD_MASK(1);
5501 val |= MVPP2_TXQ_REFILL_TOKENS_ALL_MASK;
5502 mvpp2_write(port->priv, MVPP2_TXQ_SCHED_REFILL_REG(txq->log_id), val);
5503
5504 val = MVPP2_TXQ_TOKEN_SIZE_MAX;
5505 mvpp2_write(port->priv, MVPP2_TXQ_SCHED_TOKEN_SIZE_REG(txq->log_id),
5506 val);
5507
5508 for_each_present_cpu(cpu) {
5509 txq_pcpu = per_cpu_ptr(txq->pcpu, cpu);
5510 txq_pcpu->size = txq->size;
Markus Elfring02c91ec2017-04-17 08:09:07 +02005511 txq_pcpu->buffs = kmalloc_array(txq_pcpu->size,
5512 sizeof(*txq_pcpu->buffs),
5513 GFP_KERNEL);
Thomas Petazzoni83544912016-12-21 11:28:49 +01005514 if (!txq_pcpu->buffs)
Markus Elfring20b1e162017-04-17 12:58:33 +02005515 goto cleanup;
Marcin Wojtas3f518502014-07-10 16:52:13 -03005516
5517 txq_pcpu->count = 0;
5518 txq_pcpu->reserved_num = 0;
5519 txq_pcpu->txq_put_index = 0;
5520 txq_pcpu->txq_get_index = 0;
Antoine Ténart186cd4d2017-08-23 09:46:56 +02005521
5522 txq_pcpu->tso_headers =
5523 dma_alloc_coherent(port->dev->dev.parent,
5524 MVPP2_AGGR_TXQ_SIZE * TSO_HEADER_SIZE,
5525 &txq_pcpu->tso_headers_dma,
5526 GFP_KERNEL);
5527 if (!txq_pcpu->tso_headers)
5528 goto cleanup;
Marcin Wojtas3f518502014-07-10 16:52:13 -03005529 }
5530
5531 return 0;
Markus Elfring20b1e162017-04-17 12:58:33 +02005532cleanup:
Marcin Wojtas71ce3912015-08-06 19:00:29 +02005533 for_each_present_cpu(cpu) {
5534 txq_pcpu = per_cpu_ptr(txq->pcpu, cpu);
Thomas Petazzoni83544912016-12-21 11:28:49 +01005535 kfree(txq_pcpu->buffs);
Antoine Ténart186cd4d2017-08-23 09:46:56 +02005536
5537 dma_free_coherent(port->dev->dev.parent,
5538 MVPP2_AGGR_TXQ_SIZE * MVPP2_DESC_ALIGNED_SIZE,
5539 txq_pcpu->tso_headers,
5540 txq_pcpu->tso_headers_dma);
Marcin Wojtas71ce3912015-08-06 19:00:29 +02005541 }
5542
5543 dma_free_coherent(port->dev->dev.parent,
5544 txq->size * MVPP2_DESC_ALIGNED_SIZE,
Thomas Petazzoni20396132017-03-07 16:53:00 +01005545 txq->descs, txq->descs_dma);
Marcin Wojtas71ce3912015-08-06 19:00:29 +02005546
5547 return -ENOMEM;
Marcin Wojtas3f518502014-07-10 16:52:13 -03005548}
5549
5550/* Free allocated TXQ resources */
5551static void mvpp2_txq_deinit(struct mvpp2_port *port,
5552 struct mvpp2_tx_queue *txq)
5553{
5554 struct mvpp2_txq_pcpu *txq_pcpu;
5555 int cpu;
5556
5557 for_each_present_cpu(cpu) {
5558 txq_pcpu = per_cpu_ptr(txq->pcpu, cpu);
Thomas Petazzoni83544912016-12-21 11:28:49 +01005559 kfree(txq_pcpu->buffs);
Antoine Ténart186cd4d2017-08-23 09:46:56 +02005560
5561 dma_free_coherent(port->dev->dev.parent,
5562 MVPP2_AGGR_TXQ_SIZE * MVPP2_DESC_ALIGNED_SIZE,
5563 txq_pcpu->tso_headers,
5564 txq_pcpu->tso_headers_dma);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005565 }
5566
5567 if (txq->descs)
5568 dma_free_coherent(port->dev->dev.parent,
5569 txq->size * MVPP2_DESC_ALIGNED_SIZE,
Thomas Petazzoni20396132017-03-07 16:53:00 +01005570 txq->descs, txq->descs_dma);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005571
5572 txq->descs = NULL;
5573 txq->last_desc = 0;
5574 txq->next_desc_to_proc = 0;
Thomas Petazzoni20396132017-03-07 16:53:00 +01005575 txq->descs_dma = 0;
Marcin Wojtas3f518502014-07-10 16:52:13 -03005576
5577 /* Set minimum bandwidth for disabled TXQs */
5578 mvpp2_write(port->priv, MVPP2_TXQ_SCHED_TOKEN_CNTR_REG(txq->id), 0);
5579
5580 /* Set Tx descriptors queue starting address and size */
Thomas Petazzonia704bb52017-06-10 23:18:22 +02005581 cpu = get_cpu();
Thomas Petazzonia7868412017-03-07 16:53:13 +01005582 mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_NUM_REG, txq->id);
5583 mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_DESC_ADDR_REG, 0);
5584 mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_DESC_SIZE_REG, 0);
Thomas Petazzonia704bb52017-06-10 23:18:22 +02005585 put_cpu();
Marcin Wojtas3f518502014-07-10 16:52:13 -03005586}
5587
5588/* Cleanup Tx ports */
5589static void mvpp2_txq_clean(struct mvpp2_port *port, struct mvpp2_tx_queue *txq)
5590{
5591 struct mvpp2_txq_pcpu *txq_pcpu;
5592 int delay, pending, cpu;
5593 u32 val;
5594
Thomas Petazzonia704bb52017-06-10 23:18:22 +02005595 cpu = get_cpu();
Thomas Petazzonia7868412017-03-07 16:53:13 +01005596 mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_NUM_REG, txq->id);
5597 val = mvpp2_percpu_read(port->priv, cpu, MVPP2_TXQ_PREF_BUF_REG);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005598 val |= MVPP2_TXQ_DRAIN_EN_MASK;
Thomas Petazzonia7868412017-03-07 16:53:13 +01005599 mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_PREF_BUF_REG, val);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005600
5601 /* The napi queue has been stopped so wait for all packets
5602 * to be transmitted.
5603 */
5604 delay = 0;
5605 do {
5606 if (delay >= MVPP2_TX_PENDING_TIMEOUT_MSEC) {
5607 netdev_warn(port->dev,
5608 "port %d: cleaning queue %d timed out\n",
5609 port->id, txq->log_id);
5610 break;
5611 }
5612 mdelay(1);
5613 delay++;
5614
Thomas Petazzonia7868412017-03-07 16:53:13 +01005615 pending = mvpp2_percpu_read(port->priv, cpu,
5616 MVPP2_TXQ_PENDING_REG);
5617 pending &= MVPP2_TXQ_PENDING_MASK;
Marcin Wojtas3f518502014-07-10 16:52:13 -03005618 } while (pending);
5619
5620 val &= ~MVPP2_TXQ_DRAIN_EN_MASK;
Thomas Petazzonia7868412017-03-07 16:53:13 +01005621 mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_PREF_BUF_REG, val);
Thomas Petazzonia704bb52017-06-10 23:18:22 +02005622 put_cpu();
Marcin Wojtas3f518502014-07-10 16:52:13 -03005623
5624 for_each_present_cpu(cpu) {
5625 txq_pcpu = per_cpu_ptr(txq->pcpu, cpu);
5626
5627 /* Release all packets */
5628 mvpp2_txq_bufs_free(port, txq, txq_pcpu, txq_pcpu->count);
5629
5630 /* Reset queue */
5631 txq_pcpu->count = 0;
5632 txq_pcpu->txq_put_index = 0;
5633 txq_pcpu->txq_get_index = 0;
5634 }
5635}
5636
5637/* Cleanup all Tx queues */
5638static void mvpp2_cleanup_txqs(struct mvpp2_port *port)
5639{
5640 struct mvpp2_tx_queue *txq;
5641 int queue;
5642 u32 val;
5643
5644 val = mvpp2_read(port->priv, MVPP2_TX_PORT_FLUSH_REG);
5645
5646 /* Reset Tx ports and delete Tx queues */
5647 val |= MVPP2_TX_PORT_FLUSH_MASK(port->id);
5648 mvpp2_write(port->priv, MVPP2_TX_PORT_FLUSH_REG, val);
5649
Thomas Petazzoni09f83972017-08-03 10:41:57 +02005650 for (queue = 0; queue < port->ntxqs; queue++) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03005651 txq = port->txqs[queue];
5652 mvpp2_txq_clean(port, txq);
5653 mvpp2_txq_deinit(port, txq);
5654 }
5655
5656 on_each_cpu(mvpp2_txq_sent_counter_clear, port, 1);
5657
5658 val &= ~MVPP2_TX_PORT_FLUSH_MASK(port->id);
5659 mvpp2_write(port->priv, MVPP2_TX_PORT_FLUSH_REG, val);
5660}
5661
5662/* Cleanup all Rx queues */
5663static void mvpp2_cleanup_rxqs(struct mvpp2_port *port)
5664{
5665 int queue;
5666
Thomas Petazzoni09f83972017-08-03 10:41:57 +02005667 for (queue = 0; queue < port->nrxqs; queue++)
Marcin Wojtas3f518502014-07-10 16:52:13 -03005668 mvpp2_rxq_deinit(port, port->rxqs[queue]);
5669}
5670
5671/* Init all Rx queues for port */
5672static int mvpp2_setup_rxqs(struct mvpp2_port *port)
5673{
5674 int queue, err;
5675
Thomas Petazzoni09f83972017-08-03 10:41:57 +02005676 for (queue = 0; queue < port->nrxqs; queue++) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03005677 err = mvpp2_rxq_init(port, port->rxqs[queue]);
5678 if (err)
5679 goto err_cleanup;
5680 }
5681 return 0;
5682
5683err_cleanup:
5684 mvpp2_cleanup_rxqs(port);
5685 return err;
5686}
5687
5688/* Init all tx queues for port */
5689static int mvpp2_setup_txqs(struct mvpp2_port *port)
5690{
5691 struct mvpp2_tx_queue *txq;
5692 int queue, err;
5693
Thomas Petazzoni09f83972017-08-03 10:41:57 +02005694 for (queue = 0; queue < port->ntxqs; queue++) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03005695 txq = port->txqs[queue];
5696 err = mvpp2_txq_init(port, txq);
5697 if (err)
5698 goto err_cleanup;
5699 }
5700
Thomas Petazzoni213f4282017-08-03 10:42:00 +02005701 if (port->has_tx_irqs) {
5702 mvpp2_tx_time_coal_set(port);
5703 for (queue = 0; queue < port->ntxqs; queue++) {
5704 txq = port->txqs[queue];
5705 mvpp2_tx_pkts_coal_set(port, txq);
5706 }
5707 }
5708
Marcin Wojtas3f518502014-07-10 16:52:13 -03005709 on_each_cpu(mvpp2_txq_sent_counter_clear, port, 1);
5710 return 0;
5711
5712err_cleanup:
5713 mvpp2_cleanup_txqs(port);
5714 return err;
5715}
5716
5717/* The callback for per-port interrupt */
5718static irqreturn_t mvpp2_isr(int irq, void *dev_id)
5719{
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02005720 struct mvpp2_queue_vector *qv = dev_id;
Marcin Wojtas3f518502014-07-10 16:52:13 -03005721
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02005722 mvpp2_qvec_interrupt_disable(qv);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005723
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02005724 napi_schedule(&qv->napi);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005725
5726 return IRQ_HANDLED;
5727}
5728
Antoine Tenart65a2c092017-08-30 10:29:18 +02005729static void mvpp2_gmac_set_autoneg(struct mvpp2_port *port,
5730 struct phy_device *phydev)
5731{
5732 u32 val;
5733
5734 if (port->phy_interface != PHY_INTERFACE_MODE_RGMII &&
5735 port->phy_interface != PHY_INTERFACE_MODE_RGMII_ID &&
5736 port->phy_interface != PHY_INTERFACE_MODE_RGMII_RXID &&
5737 port->phy_interface != PHY_INTERFACE_MODE_RGMII_TXID &&
5738 port->phy_interface != PHY_INTERFACE_MODE_SGMII)
5739 return;
5740
5741 val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
5742 val &= ~(MVPP2_GMAC_CONFIG_MII_SPEED |
5743 MVPP2_GMAC_CONFIG_GMII_SPEED |
5744 MVPP2_GMAC_CONFIG_FULL_DUPLEX |
5745 MVPP2_GMAC_AN_SPEED_EN |
5746 MVPP2_GMAC_AN_DUPLEX_EN);
5747
5748 if (phydev->duplex)
5749 val |= MVPP2_GMAC_CONFIG_FULL_DUPLEX;
5750
5751 if (phydev->speed == SPEED_1000)
5752 val |= MVPP2_GMAC_CONFIG_GMII_SPEED;
5753 else if (phydev->speed == SPEED_100)
5754 val |= MVPP2_GMAC_CONFIG_MII_SPEED;
5755
5756 writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
5757
5758}
5759
Marcin Wojtas3f518502014-07-10 16:52:13 -03005760/* Adjust link */
5761static void mvpp2_link_event(struct net_device *dev)
5762{
5763 struct mvpp2_port *port = netdev_priv(dev);
Philippe Reynes8e072692016-06-28 00:08:11 +02005764 struct phy_device *phydev = dev->phydev;
Antoine Tenart89273bc2017-08-30 10:29:19 +02005765 bool link_reconfigured = false;
Marcin Wojtas3f518502014-07-10 16:52:13 -03005766 u32 val;
5767
5768 if (phydev->link) {
Antoine Tenart89273bc2017-08-30 10:29:19 +02005769 if (port->phy_interface != phydev->interface && port->comphy) {
5770 /* disable current port for reconfiguration */
5771 mvpp2_interrupts_disable(port);
5772 netif_carrier_off(port->dev);
5773 mvpp2_port_disable(port);
5774 phy_power_off(port->comphy);
5775
5776 /* comphy reconfiguration */
5777 port->phy_interface = phydev->interface;
5778 mvpp22_comphy_init(port);
5779
5780 /* gop/mac reconfiguration */
5781 mvpp22_gop_init(port);
5782 mvpp2_port_mii_set(port);
5783
5784 link_reconfigured = true;
5785 }
5786
Marcin Wojtas3f518502014-07-10 16:52:13 -03005787 if ((port->speed != phydev->speed) ||
5788 (port->duplex != phydev->duplex)) {
Antoine Tenart65a2c092017-08-30 10:29:18 +02005789 mvpp2_gmac_set_autoneg(port, phydev);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005790
5791 port->duplex = phydev->duplex;
5792 port->speed = phydev->speed;
5793 }
5794 }
5795
Antoine Tenart89273bc2017-08-30 10:29:19 +02005796 if (phydev->link != port->link || link_reconfigured) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03005797 port->link = phydev->link;
Marcin Wojtas3f518502014-07-10 16:52:13 -03005798
Marcin Wojtas3f518502014-07-10 16:52:13 -03005799 if (phydev->link) {
Antoine Tenart65a2c092017-08-30 10:29:18 +02005800 if (port->phy_interface == PHY_INTERFACE_MODE_RGMII ||
5801 port->phy_interface == PHY_INTERFACE_MODE_RGMII_ID ||
5802 port->phy_interface == PHY_INTERFACE_MODE_RGMII_RXID ||
5803 port->phy_interface == PHY_INTERFACE_MODE_RGMII_TXID ||
5804 port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
5805 val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
5806 val |= (MVPP2_GMAC_FORCE_LINK_PASS |
5807 MVPP2_GMAC_FORCE_LINK_DOWN);
5808 writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
5809 }
Antoine Tenartf55744a2017-08-30 10:29:17 +02005810
5811 mvpp2_interrupts_enable(port);
5812 mvpp2_port_enable(port);
5813
Marcin Wojtas3f518502014-07-10 16:52:13 -03005814 mvpp2_egress_enable(port);
5815 mvpp2_ingress_enable(port);
Antoine Tenartf55744a2017-08-30 10:29:17 +02005816 netif_carrier_on(dev);
5817 netif_tx_wake_all_queues(dev);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005818 } else {
Antoine Tenart968b2112017-08-30 10:29:16 +02005819 port->duplex = -1;
5820 port->speed = 0;
5821
Antoine Tenartf55744a2017-08-30 10:29:17 +02005822 netif_tx_stop_all_queues(dev);
5823 netif_carrier_off(dev);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005824 mvpp2_ingress_disable(port);
5825 mvpp2_egress_disable(port);
Antoine Tenartf55744a2017-08-30 10:29:17 +02005826
5827 mvpp2_port_disable(port);
5828 mvpp2_interrupts_disable(port);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005829 }
Antoine Tenart968b2112017-08-30 10:29:16 +02005830
Marcin Wojtas3f518502014-07-10 16:52:13 -03005831 phy_print_status(phydev);
5832 }
5833}
5834
Marcin Wojtasedc660f2015-08-06 19:00:30 +02005835static void mvpp2_timer_set(struct mvpp2_port_pcpu *port_pcpu)
5836{
5837 ktime_t interval;
5838
5839 if (!port_pcpu->timer_scheduled) {
5840 port_pcpu->timer_scheduled = true;
Thomas Gleixner8b0e1952016-12-25 12:30:41 +01005841 interval = MVPP2_TXDONE_HRTIMER_PERIOD_NS;
Marcin Wojtasedc660f2015-08-06 19:00:30 +02005842 hrtimer_start(&port_pcpu->tx_done_timer, interval,
5843 HRTIMER_MODE_REL_PINNED);
5844 }
5845}
5846
5847static void mvpp2_tx_proc_cb(unsigned long data)
5848{
5849 struct net_device *dev = (struct net_device *)data;
5850 struct mvpp2_port *port = netdev_priv(dev);
5851 struct mvpp2_port_pcpu *port_pcpu = this_cpu_ptr(port->pcpu);
5852 unsigned int tx_todo, cause;
5853
5854 if (!netif_running(dev))
5855 return;
5856 port_pcpu->timer_scheduled = false;
5857
5858 /* Process all the Tx queues */
Thomas Petazzoni09f83972017-08-03 10:41:57 +02005859 cause = (1 << port->ntxqs) - 1;
Thomas Petazzoni213f4282017-08-03 10:42:00 +02005860 tx_todo = mvpp2_tx_done(port, cause, smp_processor_id());
Marcin Wojtasedc660f2015-08-06 19:00:30 +02005861
5862 /* Set the timer in case not all the packets were processed */
5863 if (tx_todo)
5864 mvpp2_timer_set(port_pcpu);
5865}
5866
5867static enum hrtimer_restart mvpp2_hr_timer_cb(struct hrtimer *timer)
5868{
5869 struct mvpp2_port_pcpu *port_pcpu = container_of(timer,
5870 struct mvpp2_port_pcpu,
5871 tx_done_timer);
5872
5873 tasklet_schedule(&port_pcpu->tx_done_tasklet);
5874
5875 return HRTIMER_NORESTART;
5876}
5877
Marcin Wojtas3f518502014-07-10 16:52:13 -03005878/* Main RX/TX processing routines */
5879
5880/* Display more error info */
5881static void mvpp2_rx_error(struct mvpp2_port *port,
5882 struct mvpp2_rx_desc *rx_desc)
5883{
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01005884 u32 status = mvpp2_rxdesc_status_get(port, rx_desc);
5885 size_t sz = mvpp2_rxdesc_size_get(port, rx_desc);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005886
5887 switch (status & MVPP2_RXD_ERR_CODE_MASK) {
5888 case MVPP2_RXD_ERR_CRC:
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01005889 netdev_err(port->dev, "bad rx status %08x (crc error), size=%zu\n",
5890 status, sz);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005891 break;
5892 case MVPP2_RXD_ERR_OVERRUN:
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01005893 netdev_err(port->dev, "bad rx status %08x (overrun error), size=%zu\n",
5894 status, sz);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005895 break;
5896 case MVPP2_RXD_ERR_RESOURCE:
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01005897 netdev_err(port->dev, "bad rx status %08x (resource error), size=%zu\n",
5898 status, sz);
Marcin Wojtas3f518502014-07-10 16:52:13 -03005899 break;
5900 }
5901}
5902
5903/* Handle RX checksum offload */
5904static void mvpp2_rx_csum(struct mvpp2_port *port, u32 status,
5905 struct sk_buff *skb)
5906{
5907 if (((status & MVPP2_RXD_L3_IP4) &&
5908 !(status & MVPP2_RXD_IP4_HEADER_ERR)) ||
5909 (status & MVPP2_RXD_L3_IP6))
5910 if (((status & MVPP2_RXD_L4_UDP) ||
5911 (status & MVPP2_RXD_L4_TCP)) &&
5912 (status & MVPP2_RXD_L4_CSUM_OK)) {
5913 skb->csum = 0;
5914 skb->ip_summed = CHECKSUM_UNNECESSARY;
5915 return;
5916 }
5917
5918 skb->ip_summed = CHECKSUM_NONE;
5919}
5920
5921/* Reuse skb if possible, or allocate a new skb and add it to BM pool */
5922static int mvpp2_rx_refill(struct mvpp2_port *port,
Thomas Petazzoni56b8aae2017-06-10 23:18:21 +02005923 struct mvpp2_bm_pool *bm_pool, int pool)
Marcin Wojtas3f518502014-07-10 16:52:13 -03005924{
Thomas Petazzoni20396132017-03-07 16:53:00 +01005925 dma_addr_t dma_addr;
Thomas Petazzoni4e4a1052017-03-07 16:53:04 +01005926 phys_addr_t phys_addr;
Thomas Petazzoni0e037282017-02-21 11:28:12 +01005927 void *buf;
Marcin Wojtas3f518502014-07-10 16:52:13 -03005928
Marcin Wojtas3f518502014-07-10 16:52:13 -03005929 /* No recycle or too many buffers are in use, so allocate a new skb */
Thomas Petazzoni4e4a1052017-03-07 16:53:04 +01005930 buf = mvpp2_buf_alloc(port, bm_pool, &dma_addr, &phys_addr,
5931 GFP_ATOMIC);
Thomas Petazzoni0e037282017-02-21 11:28:12 +01005932 if (!buf)
Marcin Wojtas3f518502014-07-10 16:52:13 -03005933 return -ENOMEM;
5934
Thomas Petazzoni7d7627b2017-06-22 14:23:20 +02005935 mvpp2_bm_pool_put(port, pool, dma_addr, phys_addr);
Thomas Petazzoni7ef7e1d2017-02-21 11:28:07 +01005936
Marcin Wojtas3f518502014-07-10 16:52:13 -03005937 return 0;
5938}
5939
5940/* Handle tx checksum */
5941static u32 mvpp2_skb_tx_csum(struct mvpp2_port *port, struct sk_buff *skb)
5942{
5943 if (skb->ip_summed == CHECKSUM_PARTIAL) {
5944 int ip_hdr_len = 0;
5945 u8 l4_proto;
5946
5947 if (skb->protocol == htons(ETH_P_IP)) {
5948 struct iphdr *ip4h = ip_hdr(skb);
5949
5950 /* Calculate IPv4 checksum and L4 checksum */
5951 ip_hdr_len = ip4h->ihl;
5952 l4_proto = ip4h->protocol;
5953 } else if (skb->protocol == htons(ETH_P_IPV6)) {
5954 struct ipv6hdr *ip6h = ipv6_hdr(skb);
5955
5956 /* Read l4_protocol from one of IPv6 extra headers */
5957 if (skb_network_header_len(skb) > 0)
5958 ip_hdr_len = (skb_network_header_len(skb) >> 2);
5959 l4_proto = ip6h->nexthdr;
5960 } else {
5961 return MVPP2_TXD_L4_CSUM_NOT;
5962 }
5963
5964 return mvpp2_txq_desc_csum(skb_network_offset(skb),
5965 skb->protocol, ip_hdr_len, l4_proto);
5966 }
5967
5968 return MVPP2_TXD_L4_CSUM_NOT | MVPP2_TXD_IP_CSUM_DISABLE;
5969}
5970
Marcin Wojtas3f518502014-07-10 16:52:13 -03005971/* Main rx processing */
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02005972static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi,
5973 int rx_todo, struct mvpp2_rx_queue *rxq)
Marcin Wojtas3f518502014-07-10 16:52:13 -03005974{
5975 struct net_device *dev = port->dev;
Marcin Wojtasb5015852015-12-03 15:20:51 +01005976 int rx_received;
5977 int rx_done = 0;
Marcin Wojtas3f518502014-07-10 16:52:13 -03005978 u32 rcvd_pkts = 0;
5979 u32 rcvd_bytes = 0;
5980
5981 /* Get number of received packets and clamp the to-do */
5982 rx_received = mvpp2_rxq_received(port, rxq->id);
5983 if (rx_todo > rx_received)
5984 rx_todo = rx_received;
5985
Marcin Wojtasb5015852015-12-03 15:20:51 +01005986 while (rx_done < rx_todo) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03005987 struct mvpp2_rx_desc *rx_desc = mvpp2_rxq_next_desc_get(rxq);
5988 struct mvpp2_bm_pool *bm_pool;
5989 struct sk_buff *skb;
Thomas Petazzoni0e037282017-02-21 11:28:12 +01005990 unsigned int frag_size;
Thomas Petazzoni20396132017-03-07 16:53:00 +01005991 dma_addr_t dma_addr;
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01005992 phys_addr_t phys_addr;
Thomas Petazzoni56b8aae2017-06-10 23:18:21 +02005993 u32 rx_status;
Marcin Wojtas3f518502014-07-10 16:52:13 -03005994 int pool, rx_bytes, err;
Thomas Petazzoni0e037282017-02-21 11:28:12 +01005995 void *data;
Marcin Wojtas3f518502014-07-10 16:52:13 -03005996
Marcin Wojtasb5015852015-12-03 15:20:51 +01005997 rx_done++;
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01005998 rx_status = mvpp2_rxdesc_status_get(port, rx_desc);
5999 rx_bytes = mvpp2_rxdesc_size_get(port, rx_desc);
6000 rx_bytes -= MVPP2_MH_SIZE;
6001 dma_addr = mvpp2_rxdesc_dma_addr_get(port, rx_desc);
6002 phys_addr = mvpp2_rxdesc_cookie_get(port, rx_desc);
6003 data = (void *)phys_to_virt(phys_addr);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006004
Thomas Petazzoni56b8aae2017-06-10 23:18:21 +02006005 pool = (rx_status & MVPP2_RXD_BM_POOL_ID_MASK) >>
6006 MVPP2_RXD_BM_POOL_ID_OFFS;
Marcin Wojtas3f518502014-07-10 16:52:13 -03006007 bm_pool = &port->priv->bm_pools[pool];
Marcin Wojtas3f518502014-07-10 16:52:13 -03006008
6009 /* In case of an error, release the requested buffer pointer
6010 * to the Buffer Manager. This request process is controlled
6011 * by the hardware, and the information about the buffer is
6012 * comprised by the RX descriptor.
6013 */
6014 if (rx_status & MVPP2_RXD_ERR_SUMMARY) {
Markus Elfring8a524882017-04-17 10:52:02 +02006015err_drop_frame:
Marcin Wojtas3f518502014-07-10 16:52:13 -03006016 dev->stats.rx_errors++;
6017 mvpp2_rx_error(port, rx_desc);
Marcin Wojtasb5015852015-12-03 15:20:51 +01006018 /* Return the buffer to the pool */
Thomas Petazzoni7d7627b2017-06-22 14:23:20 +02006019 mvpp2_bm_pool_put(port, pool, dma_addr, phys_addr);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006020 continue;
6021 }
6022
Thomas Petazzoni0e037282017-02-21 11:28:12 +01006023 if (bm_pool->frag_size > PAGE_SIZE)
6024 frag_size = 0;
6025 else
6026 frag_size = bm_pool->frag_size;
6027
6028 skb = build_skb(data, frag_size);
6029 if (!skb) {
6030 netdev_warn(port->dev, "skb build failed\n");
6031 goto err_drop_frame;
6032 }
Marcin Wojtas3f518502014-07-10 16:52:13 -03006033
Thomas Petazzoni56b8aae2017-06-10 23:18:21 +02006034 err = mvpp2_rx_refill(port, bm_pool, pool);
Marcin Wojtasb5015852015-12-03 15:20:51 +01006035 if (err) {
6036 netdev_err(port->dev, "failed to refill BM pools\n");
6037 goto err_drop_frame;
6038 }
6039
Thomas Petazzoni20396132017-03-07 16:53:00 +01006040 dma_unmap_single(dev->dev.parent, dma_addr,
Marcin Wojtas4229d502015-12-03 15:20:50 +01006041 bm_pool->buf_size, DMA_FROM_DEVICE);
6042
Marcin Wojtas3f518502014-07-10 16:52:13 -03006043 rcvd_pkts++;
6044 rcvd_bytes += rx_bytes;
Marcin Wojtas3f518502014-07-10 16:52:13 -03006045
Thomas Petazzoni0e037282017-02-21 11:28:12 +01006046 skb_reserve(skb, MVPP2_MH_SIZE + NET_SKB_PAD);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006047 skb_put(skb, rx_bytes);
6048 skb->protocol = eth_type_trans(skb, dev);
6049 mvpp2_rx_csum(port, rx_status, skb);
6050
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006051 napi_gro_receive(napi, skb);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006052 }
6053
6054 if (rcvd_pkts) {
6055 struct mvpp2_pcpu_stats *stats = this_cpu_ptr(port->stats);
6056
6057 u64_stats_update_begin(&stats->syncp);
6058 stats->rx_packets += rcvd_pkts;
6059 stats->rx_bytes += rcvd_bytes;
6060 u64_stats_update_end(&stats->syncp);
6061 }
6062
6063 /* Update Rx queue management counters */
6064 wmb();
Marcin Wojtasb5015852015-12-03 15:20:51 +01006065 mvpp2_rxq_status_update(port, rxq->id, rx_done, rx_done);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006066
6067 return rx_todo;
6068}
6069
6070static inline void
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01006071tx_desc_unmap_put(struct mvpp2_port *port, struct mvpp2_tx_queue *txq,
Marcin Wojtas3f518502014-07-10 16:52:13 -03006072 struct mvpp2_tx_desc *desc)
6073{
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01006074 dma_addr_t buf_dma_addr =
6075 mvpp2_txdesc_dma_addr_get(port, desc);
6076 size_t buf_sz =
6077 mvpp2_txdesc_size_get(port, desc);
6078 dma_unmap_single(port->dev->dev.parent, buf_dma_addr,
6079 buf_sz, DMA_TO_DEVICE);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006080 mvpp2_txq_desc_put(txq);
6081}
6082
6083/* Handle tx fragmentation processing */
6084static int mvpp2_tx_frag_process(struct mvpp2_port *port, struct sk_buff *skb,
6085 struct mvpp2_tx_queue *aggr_txq,
6086 struct mvpp2_tx_queue *txq)
6087{
6088 struct mvpp2_txq_pcpu *txq_pcpu = this_cpu_ptr(txq->pcpu);
6089 struct mvpp2_tx_desc *tx_desc;
6090 int i;
Thomas Petazzoni20396132017-03-07 16:53:00 +01006091 dma_addr_t buf_dma_addr;
Marcin Wojtas3f518502014-07-10 16:52:13 -03006092
6093 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
6094 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
6095 void *addr = page_address(frag->page.p) + frag->page_offset;
6096
6097 tx_desc = mvpp2_txq_next_desc_get(aggr_txq);
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01006098 mvpp2_txdesc_txq_set(port, tx_desc, txq->id);
6099 mvpp2_txdesc_size_set(port, tx_desc, frag->size);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006100
Thomas Petazzoni20396132017-03-07 16:53:00 +01006101 buf_dma_addr = dma_map_single(port->dev->dev.parent, addr,
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01006102 frag->size,
6103 DMA_TO_DEVICE);
Thomas Petazzoni20396132017-03-07 16:53:00 +01006104 if (dma_mapping_error(port->dev->dev.parent, buf_dma_addr)) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03006105 mvpp2_txq_desc_put(txq);
Markus Elfring32bae632017-04-17 11:36:34 +02006106 goto cleanup;
Marcin Wojtas3f518502014-07-10 16:52:13 -03006107 }
6108
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01006109 mvpp2_txdesc_offset_set(port, tx_desc,
6110 buf_dma_addr & MVPP2_TX_DESC_ALIGN);
6111 mvpp2_txdesc_dma_addr_set(port, tx_desc,
6112 buf_dma_addr & ~MVPP2_TX_DESC_ALIGN);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006113
6114 if (i == (skb_shinfo(skb)->nr_frags - 1)) {
6115 /* Last descriptor */
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01006116 mvpp2_txdesc_cmd_set(port, tx_desc,
6117 MVPP2_TXD_L_DESC);
6118 mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006119 } else {
6120 /* Descriptor in the middle: Not First, Not Last */
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01006121 mvpp2_txdesc_cmd_set(port, tx_desc, 0);
6122 mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006123 }
6124 }
6125
6126 return 0;
Markus Elfring32bae632017-04-17 11:36:34 +02006127cleanup:
Marcin Wojtas3f518502014-07-10 16:52:13 -03006128 /* Release all descriptors that were used to map fragments of
6129 * this packet, as well as the corresponding DMA mappings
6130 */
6131 for (i = i - 1; i >= 0; i--) {
6132 tx_desc = txq->descs + i;
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01006133 tx_desc_unmap_put(port, txq, tx_desc);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006134 }
6135
6136 return -ENOMEM;
6137}
6138
Antoine Ténart186cd4d2017-08-23 09:46:56 +02006139static inline void mvpp2_tso_put_hdr(struct sk_buff *skb,
6140 struct net_device *dev,
6141 struct mvpp2_tx_queue *txq,
6142 struct mvpp2_tx_queue *aggr_txq,
6143 struct mvpp2_txq_pcpu *txq_pcpu,
6144 int hdr_sz)
6145{
6146 struct mvpp2_port *port = netdev_priv(dev);
6147 struct mvpp2_tx_desc *tx_desc = mvpp2_txq_next_desc_get(aggr_txq);
6148 dma_addr_t addr;
6149
6150 mvpp2_txdesc_txq_set(port, tx_desc, txq->id);
6151 mvpp2_txdesc_size_set(port, tx_desc, hdr_sz);
6152
6153 addr = txq_pcpu->tso_headers_dma +
6154 txq_pcpu->txq_put_index * TSO_HEADER_SIZE;
6155 mvpp2_txdesc_offset_set(port, tx_desc, addr & MVPP2_TX_DESC_ALIGN);
6156 mvpp2_txdesc_dma_addr_set(port, tx_desc, addr & ~MVPP2_TX_DESC_ALIGN);
6157
6158 mvpp2_txdesc_cmd_set(port, tx_desc, mvpp2_skb_tx_csum(port, skb) |
6159 MVPP2_TXD_F_DESC |
6160 MVPP2_TXD_PADDING_DISABLE);
6161 mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);
6162}
6163
6164static inline int mvpp2_tso_put_data(struct sk_buff *skb,
6165 struct net_device *dev, struct tso_t *tso,
6166 struct mvpp2_tx_queue *txq,
6167 struct mvpp2_tx_queue *aggr_txq,
6168 struct mvpp2_txq_pcpu *txq_pcpu,
6169 int sz, bool left, bool last)
6170{
6171 struct mvpp2_port *port = netdev_priv(dev);
6172 struct mvpp2_tx_desc *tx_desc = mvpp2_txq_next_desc_get(aggr_txq);
6173 dma_addr_t buf_dma_addr;
6174
6175 mvpp2_txdesc_txq_set(port, tx_desc, txq->id);
6176 mvpp2_txdesc_size_set(port, tx_desc, sz);
6177
6178 buf_dma_addr = dma_map_single(dev->dev.parent, tso->data, sz,
6179 DMA_TO_DEVICE);
6180 if (unlikely(dma_mapping_error(dev->dev.parent, buf_dma_addr))) {
6181 mvpp2_txq_desc_put(txq);
6182 return -ENOMEM;
6183 }
6184
6185 mvpp2_txdesc_offset_set(port, tx_desc,
6186 buf_dma_addr & MVPP2_TX_DESC_ALIGN);
6187 mvpp2_txdesc_dma_addr_set(port, tx_desc,
6188 buf_dma_addr & ~MVPP2_TX_DESC_ALIGN);
6189
6190 if (!left) {
6191 mvpp2_txdesc_cmd_set(port, tx_desc, MVPP2_TXD_L_DESC);
6192 if (last) {
6193 mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc);
6194 return 0;
6195 }
6196 } else {
6197 mvpp2_txdesc_cmd_set(port, tx_desc, 0);
6198 }
6199
6200 mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);
6201 return 0;
6202}
6203
6204static int mvpp2_tx_tso(struct sk_buff *skb, struct net_device *dev,
6205 struct mvpp2_tx_queue *txq,
6206 struct mvpp2_tx_queue *aggr_txq,
6207 struct mvpp2_txq_pcpu *txq_pcpu)
6208{
6209 struct mvpp2_port *port = netdev_priv(dev);
6210 struct tso_t tso;
6211 int hdr_sz = skb_transport_offset(skb) + tcp_hdrlen(skb);
6212 int i, len, descs = 0;
6213
6214 /* Check number of available descriptors */
6215 if (mvpp2_aggr_desc_num_check(port->priv, aggr_txq,
6216 tso_count_descs(skb)) ||
6217 mvpp2_txq_reserved_desc_num_proc(port->priv, txq, txq_pcpu,
6218 tso_count_descs(skb)))
6219 return 0;
6220
6221 tso_start(skb, &tso);
6222 len = skb->len - hdr_sz;
6223 while (len > 0) {
6224 int left = min_t(int, skb_shinfo(skb)->gso_size, len);
6225 char *hdr = txq_pcpu->tso_headers +
6226 txq_pcpu->txq_put_index * TSO_HEADER_SIZE;
6227
6228 len -= left;
6229 descs++;
6230
6231 tso_build_hdr(skb, hdr, &tso, left, len == 0);
6232 mvpp2_tso_put_hdr(skb, dev, txq, aggr_txq, txq_pcpu, hdr_sz);
6233
6234 while (left > 0) {
6235 int sz = min_t(int, tso.size, left);
6236 left -= sz;
6237 descs++;
6238
6239 if (mvpp2_tso_put_data(skb, dev, &tso, txq, aggr_txq,
6240 txq_pcpu, sz, left, len == 0))
6241 goto release;
6242 tso_build_data(skb, &tso, sz);
6243 }
6244 }
6245
6246 return descs;
6247
6248release:
6249 for (i = descs - 1; i >= 0; i--) {
6250 struct mvpp2_tx_desc *tx_desc = txq->descs + i;
6251 tx_desc_unmap_put(port, txq, tx_desc);
6252 }
6253 return 0;
6254}
6255
Marcin Wojtas3f518502014-07-10 16:52:13 -03006256/* Main tx processing */
6257static int mvpp2_tx(struct sk_buff *skb, struct net_device *dev)
6258{
6259 struct mvpp2_port *port = netdev_priv(dev);
6260 struct mvpp2_tx_queue *txq, *aggr_txq;
6261 struct mvpp2_txq_pcpu *txq_pcpu;
6262 struct mvpp2_tx_desc *tx_desc;
Thomas Petazzoni20396132017-03-07 16:53:00 +01006263 dma_addr_t buf_dma_addr;
Marcin Wojtas3f518502014-07-10 16:52:13 -03006264 int frags = 0;
6265 u16 txq_id;
6266 u32 tx_cmd;
6267
6268 txq_id = skb_get_queue_mapping(skb);
6269 txq = port->txqs[txq_id];
6270 txq_pcpu = this_cpu_ptr(txq->pcpu);
6271 aggr_txq = &port->priv->aggr_txqs[smp_processor_id()];
6272
Antoine Ténart186cd4d2017-08-23 09:46:56 +02006273 if (skb_is_gso(skb)) {
6274 frags = mvpp2_tx_tso(skb, dev, txq, aggr_txq, txq_pcpu);
6275 goto out;
6276 }
Marcin Wojtas3f518502014-07-10 16:52:13 -03006277 frags = skb_shinfo(skb)->nr_frags + 1;
6278
6279 /* Check number of available descriptors */
6280 if (mvpp2_aggr_desc_num_check(port->priv, aggr_txq, frags) ||
6281 mvpp2_txq_reserved_desc_num_proc(port->priv, txq,
6282 txq_pcpu, frags)) {
6283 frags = 0;
6284 goto out;
6285 }
6286
6287 /* Get a descriptor for the first part of the packet */
6288 tx_desc = mvpp2_txq_next_desc_get(aggr_txq);
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01006289 mvpp2_txdesc_txq_set(port, tx_desc, txq->id);
6290 mvpp2_txdesc_size_set(port, tx_desc, skb_headlen(skb));
Marcin Wojtas3f518502014-07-10 16:52:13 -03006291
Thomas Petazzoni20396132017-03-07 16:53:00 +01006292 buf_dma_addr = dma_map_single(dev->dev.parent, skb->data,
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01006293 skb_headlen(skb), DMA_TO_DEVICE);
Thomas Petazzoni20396132017-03-07 16:53:00 +01006294 if (unlikely(dma_mapping_error(dev->dev.parent, buf_dma_addr))) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03006295 mvpp2_txq_desc_put(txq);
6296 frags = 0;
6297 goto out;
6298 }
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01006299
6300 mvpp2_txdesc_offset_set(port, tx_desc,
6301 buf_dma_addr & MVPP2_TX_DESC_ALIGN);
6302 mvpp2_txdesc_dma_addr_set(port, tx_desc,
6303 buf_dma_addr & ~MVPP2_TX_DESC_ALIGN);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006304
6305 tx_cmd = mvpp2_skb_tx_csum(port, skb);
6306
6307 if (frags == 1) {
6308 /* First and Last descriptor */
6309 tx_cmd |= MVPP2_TXD_F_DESC | MVPP2_TXD_L_DESC;
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01006310 mvpp2_txdesc_cmd_set(port, tx_desc, tx_cmd);
6311 mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006312 } else {
6313 /* First but not Last */
6314 tx_cmd |= MVPP2_TXD_F_DESC | MVPP2_TXD_PADDING_DISABLE;
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01006315 mvpp2_txdesc_cmd_set(port, tx_desc, tx_cmd);
6316 mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006317
6318 /* Continue with other skb fragments */
6319 if (mvpp2_tx_frag_process(port, skb, aggr_txq, txq)) {
Thomas Petazzoniac3dd2772017-03-07 16:53:05 +01006320 tx_desc_unmap_put(port, txq, tx_desc);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006321 frags = 0;
6322 goto out;
6323 }
6324 }
6325
Marcin Wojtas3f518502014-07-10 16:52:13 -03006326out:
6327 if (frags > 0) {
6328 struct mvpp2_pcpu_stats *stats = this_cpu_ptr(port->stats);
Antoine Ténart186cd4d2017-08-23 09:46:56 +02006329 struct netdev_queue *nq = netdev_get_tx_queue(dev, txq_id);
6330
6331 txq_pcpu->reserved_num -= frags;
6332 txq_pcpu->count += frags;
6333 aggr_txq->count += frags;
6334
6335 /* Enable transmit */
6336 wmb();
6337 mvpp2_aggr_txq_pend_desc_add(port, frags);
6338
6339 if (txq_pcpu->size - txq_pcpu->count < MAX_SKB_FRAGS + 1)
6340 netif_tx_stop_queue(nq);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006341
6342 u64_stats_update_begin(&stats->syncp);
6343 stats->tx_packets++;
6344 stats->tx_bytes += skb->len;
6345 u64_stats_update_end(&stats->syncp);
6346 } else {
6347 dev->stats.tx_dropped++;
6348 dev_kfree_skb_any(skb);
6349 }
6350
Marcin Wojtasedc660f2015-08-06 19:00:30 +02006351 /* Finalize TX processing */
6352 if (txq_pcpu->count >= txq->done_pkts_coal)
6353 mvpp2_txq_done(port, txq, txq_pcpu);
6354
6355 /* Set the timer in case not all frags were processed */
Thomas Petazzoni213f4282017-08-03 10:42:00 +02006356 if (!port->has_tx_irqs && txq_pcpu->count <= frags &&
6357 txq_pcpu->count > 0) {
Marcin Wojtasedc660f2015-08-06 19:00:30 +02006358 struct mvpp2_port_pcpu *port_pcpu = this_cpu_ptr(port->pcpu);
6359
6360 mvpp2_timer_set(port_pcpu);
6361 }
6362
Marcin Wojtas3f518502014-07-10 16:52:13 -03006363 return NETDEV_TX_OK;
6364}
6365
6366static inline void mvpp2_cause_error(struct net_device *dev, int cause)
6367{
6368 if (cause & MVPP2_CAUSE_FCS_ERR_MASK)
6369 netdev_err(dev, "FCS error\n");
6370 if (cause & MVPP2_CAUSE_RX_FIFO_OVERRUN_MASK)
6371 netdev_err(dev, "rx fifo overrun error\n");
6372 if (cause & MVPP2_CAUSE_TX_FIFO_UNDERRUN_MASK)
6373 netdev_err(dev, "tx fifo underrun error\n");
6374}
6375
Marcin Wojtasedc660f2015-08-06 19:00:30 +02006376static int mvpp2_poll(struct napi_struct *napi, int budget)
Marcin Wojtas3f518502014-07-10 16:52:13 -03006377{
Thomas Petazzoni213f4282017-08-03 10:42:00 +02006378 u32 cause_rx_tx, cause_rx, cause_tx, cause_misc;
Marcin Wojtasedc660f2015-08-06 19:00:30 +02006379 int rx_done = 0;
6380 struct mvpp2_port *port = netdev_priv(napi->dev);
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006381 struct mvpp2_queue_vector *qv;
Thomas Petazzonia7868412017-03-07 16:53:13 +01006382 int cpu = smp_processor_id();
Marcin Wojtas3f518502014-07-10 16:52:13 -03006383
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006384 qv = container_of(napi, struct mvpp2_queue_vector, napi);
6385
Marcin Wojtas3f518502014-07-10 16:52:13 -03006386 /* Rx/Tx cause register
6387 *
6388 * Bits 0-15: each bit indicates received packets on the Rx queue
6389 * (bit 0 is for Rx queue 0).
6390 *
6391 * Bits 16-23: each bit indicates transmitted packets on the Tx queue
6392 * (bit 16 is for Tx queue 0).
6393 *
6394 * Each CPU has its own Rx/Tx cause register
6395 */
Thomas Petazzoni213f4282017-08-03 10:42:00 +02006396 cause_rx_tx = mvpp2_percpu_read(port->priv, qv->sw_thread_id,
Thomas Petazzonia7868412017-03-07 16:53:13 +01006397 MVPP2_ISR_RX_TX_CAUSE_REG(port->id));
Marcin Wojtas3f518502014-07-10 16:52:13 -03006398
Thomas Petazzoni213f4282017-08-03 10:42:00 +02006399 cause_misc = cause_rx_tx & MVPP2_CAUSE_MISC_SUM_MASK;
Marcin Wojtas3f518502014-07-10 16:52:13 -03006400 if (cause_misc) {
6401 mvpp2_cause_error(port->dev, cause_misc);
6402
6403 /* Clear the cause register */
6404 mvpp2_write(port->priv, MVPP2_ISR_MISC_CAUSE_REG, 0);
Thomas Petazzonia7868412017-03-07 16:53:13 +01006405 mvpp2_percpu_write(port->priv, cpu,
6406 MVPP2_ISR_RX_TX_CAUSE_REG(port->id),
6407 cause_rx_tx & ~MVPP2_CAUSE_MISC_SUM_MASK);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006408 }
6409
Thomas Petazzoni213f4282017-08-03 10:42:00 +02006410 cause_tx = cause_rx_tx & MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_MASK;
6411 if (cause_tx) {
6412 cause_tx >>= MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_OFFSET;
6413 mvpp2_tx_done(port, cause_tx, qv->sw_thread_id);
6414 }
Marcin Wojtas3f518502014-07-10 16:52:13 -03006415
6416 /* Process RX packets */
Thomas Petazzoni213f4282017-08-03 10:42:00 +02006417 cause_rx = cause_rx_tx & MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK;
6418 cause_rx <<= qv->first_rxq;
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006419 cause_rx |= qv->pending_cause_rx;
Marcin Wojtas3f518502014-07-10 16:52:13 -03006420 while (cause_rx && budget > 0) {
6421 int count;
6422 struct mvpp2_rx_queue *rxq;
6423
6424 rxq = mvpp2_get_rx_queue(port, cause_rx);
6425 if (!rxq)
6426 break;
6427
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006428 count = mvpp2_rx(port, napi, budget, rxq);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006429 rx_done += count;
6430 budget -= count;
6431 if (budget > 0) {
6432 /* Clear the bit associated to this Rx queue
6433 * so that next iteration will continue from
6434 * the next Rx queue.
6435 */
6436 cause_rx &= ~(1 << rxq->logic_rxq);
6437 }
6438 }
6439
6440 if (budget > 0) {
6441 cause_rx = 0;
Eric Dumazet6ad20162017-01-30 08:22:01 -08006442 napi_complete_done(napi, rx_done);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006443
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006444 mvpp2_qvec_interrupt_enable(qv);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006445 }
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006446 qv->pending_cause_rx = cause_rx;
Marcin Wojtas3f518502014-07-10 16:52:13 -03006447 return rx_done;
6448}
6449
6450/* Set hw internals when starting port */
6451static void mvpp2_start_dev(struct mvpp2_port *port)
6452{
Philippe Reynes8e072692016-06-28 00:08:11 +02006453 struct net_device *ndev = port->dev;
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006454 int i;
Philippe Reynes8e072692016-06-28 00:08:11 +02006455
Stefan Chulski76eb1b12017-08-22 19:08:26 +02006456 if (port->gop_id == 0 &&
6457 (port->phy_interface == PHY_INTERFACE_MODE_XAUI ||
6458 port->phy_interface == PHY_INTERFACE_MODE_10GKR))
6459 mvpp2_xlg_max_rx_size_set(port);
6460 else
6461 mvpp2_gmac_max_rx_size_set(port);
6462
Marcin Wojtas3f518502014-07-10 16:52:13 -03006463 mvpp2_txp_max_tx_size_set(port);
6464
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006465 for (i = 0; i < port->nqvecs; i++)
6466 napi_enable(&port->qvecs[i].napi);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006467
6468 /* Enable interrupts on all CPUs */
6469 mvpp2_interrupts_enable(port);
6470
Antoine Tenart542897d2017-08-30 10:29:15 +02006471 if (port->priv->hw_version == MVPP22) {
6472 mvpp22_comphy_init(port);
Antoine Ténartf84bf382017-08-22 19:08:27 +02006473 mvpp22_gop_init(port);
Antoine Tenart542897d2017-08-30 10:29:15 +02006474 }
Antoine Ténartf84bf382017-08-22 19:08:27 +02006475
Antoine Ténart2055d622017-08-22 19:08:23 +02006476 mvpp2_port_mii_set(port);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006477 mvpp2_port_enable(port);
Philippe Reynes8e072692016-06-28 00:08:11 +02006478 phy_start(ndev->phydev);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006479 netif_tx_start_all_queues(port->dev);
6480}
6481
6482/* Set hw internals when stopping port */
6483static void mvpp2_stop_dev(struct mvpp2_port *port)
6484{
Philippe Reynes8e072692016-06-28 00:08:11 +02006485 struct net_device *ndev = port->dev;
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006486 int i;
Philippe Reynes8e072692016-06-28 00:08:11 +02006487
Marcin Wojtas3f518502014-07-10 16:52:13 -03006488 /* Stop new packets from arriving to RXQs */
6489 mvpp2_ingress_disable(port);
6490
6491 mdelay(10);
6492
6493 /* Disable interrupts on all CPUs */
6494 mvpp2_interrupts_disable(port);
6495
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006496 for (i = 0; i < port->nqvecs; i++)
6497 napi_disable(&port->qvecs[i].napi);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006498
6499 netif_carrier_off(port->dev);
6500 netif_tx_stop_all_queues(port->dev);
6501
6502 mvpp2_egress_disable(port);
6503 mvpp2_port_disable(port);
Philippe Reynes8e072692016-06-28 00:08:11 +02006504 phy_stop(ndev->phydev);
Antoine Tenart542897d2017-08-30 10:29:15 +02006505 phy_power_off(port->comphy);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006506}
6507
Marcin Wojtas3f518502014-07-10 16:52:13 -03006508static int mvpp2_check_ringparam_valid(struct net_device *dev,
6509 struct ethtool_ringparam *ring)
6510{
6511 u16 new_rx_pending = ring->rx_pending;
6512 u16 new_tx_pending = ring->tx_pending;
6513
6514 if (ring->rx_pending == 0 || ring->tx_pending == 0)
6515 return -EINVAL;
6516
6517 if (ring->rx_pending > MVPP2_MAX_RXD)
6518 new_rx_pending = MVPP2_MAX_RXD;
6519 else if (!IS_ALIGNED(ring->rx_pending, 16))
6520 new_rx_pending = ALIGN(ring->rx_pending, 16);
6521
6522 if (ring->tx_pending > MVPP2_MAX_TXD)
6523 new_tx_pending = MVPP2_MAX_TXD;
6524 else if (!IS_ALIGNED(ring->tx_pending, 32))
6525 new_tx_pending = ALIGN(ring->tx_pending, 32);
6526
6527 if (ring->rx_pending != new_rx_pending) {
6528 netdev_info(dev, "illegal Rx ring size value %d, round to %d\n",
6529 ring->rx_pending, new_rx_pending);
6530 ring->rx_pending = new_rx_pending;
6531 }
6532
6533 if (ring->tx_pending != new_tx_pending) {
6534 netdev_info(dev, "illegal Tx ring size value %d, round to %d\n",
6535 ring->tx_pending, new_tx_pending);
6536 ring->tx_pending = new_tx_pending;
6537 }
6538
6539 return 0;
6540}
6541
Thomas Petazzoni26975822017-03-07 16:53:14 +01006542static void mvpp21_get_mac_address(struct mvpp2_port *port, unsigned char *addr)
Marcin Wojtas3f518502014-07-10 16:52:13 -03006543{
6544 u32 mac_addr_l, mac_addr_m, mac_addr_h;
6545
6546 mac_addr_l = readl(port->base + MVPP2_GMAC_CTRL_1_REG);
6547 mac_addr_m = readl(port->priv->lms_base + MVPP2_SRC_ADDR_MIDDLE);
6548 mac_addr_h = readl(port->priv->lms_base + MVPP2_SRC_ADDR_HIGH);
6549 addr[0] = (mac_addr_h >> 24) & 0xFF;
6550 addr[1] = (mac_addr_h >> 16) & 0xFF;
6551 addr[2] = (mac_addr_h >> 8) & 0xFF;
6552 addr[3] = mac_addr_h & 0xFF;
6553 addr[4] = mac_addr_m & 0xFF;
6554 addr[5] = (mac_addr_l >> MVPP2_GMAC_SA_LOW_OFFS) & 0xFF;
6555}
6556
6557static int mvpp2_phy_connect(struct mvpp2_port *port)
6558{
6559 struct phy_device *phy_dev;
6560
6561 phy_dev = of_phy_connect(port->dev, port->phy_node, mvpp2_link_event, 0,
6562 port->phy_interface);
6563 if (!phy_dev) {
6564 netdev_err(port->dev, "cannot connect to phy\n");
6565 return -ENODEV;
6566 }
6567 phy_dev->supported &= PHY_GBIT_FEATURES;
6568 phy_dev->advertising = phy_dev->supported;
6569
Marcin Wojtas3f518502014-07-10 16:52:13 -03006570 port->link = 0;
6571 port->duplex = 0;
6572 port->speed = 0;
6573
6574 return 0;
6575}
6576
6577static void mvpp2_phy_disconnect(struct mvpp2_port *port)
6578{
Philippe Reynes8e072692016-06-28 00:08:11 +02006579 struct net_device *ndev = port->dev;
6580
6581 phy_disconnect(ndev->phydev);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006582}
6583
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006584static int mvpp2_irqs_init(struct mvpp2_port *port)
6585{
6586 int err, i;
6587
6588 for (i = 0; i < port->nqvecs; i++) {
6589 struct mvpp2_queue_vector *qv = port->qvecs + i;
6590
6591 err = request_irq(qv->irq, mvpp2_isr, 0, port->dev->name, qv);
6592 if (err)
6593 goto err;
Thomas Petazzoni213f4282017-08-03 10:42:00 +02006594
6595 if (qv->type == MVPP2_QUEUE_VECTOR_PRIVATE)
6596 irq_set_affinity_hint(qv->irq,
6597 cpumask_of(qv->sw_thread_id));
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006598 }
6599
6600 return 0;
6601err:
6602 for (i = 0; i < port->nqvecs; i++) {
6603 struct mvpp2_queue_vector *qv = port->qvecs + i;
6604
Thomas Petazzoni213f4282017-08-03 10:42:00 +02006605 irq_set_affinity_hint(qv->irq, NULL);
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006606 free_irq(qv->irq, qv);
6607 }
6608
6609 return err;
6610}
6611
6612static void mvpp2_irqs_deinit(struct mvpp2_port *port)
6613{
6614 int i;
6615
6616 for (i = 0; i < port->nqvecs; i++) {
6617 struct mvpp2_queue_vector *qv = port->qvecs + i;
6618
Thomas Petazzoni213f4282017-08-03 10:42:00 +02006619 irq_set_affinity_hint(qv->irq, NULL);
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006620 free_irq(qv->irq, qv);
6621 }
6622}
6623
Marcin Wojtas3f518502014-07-10 16:52:13 -03006624static int mvpp2_open(struct net_device *dev)
6625{
6626 struct mvpp2_port *port = netdev_priv(dev);
6627 unsigned char mac_bcast[ETH_ALEN] = {
6628 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
6629 int err;
6630
6631 err = mvpp2_prs_mac_da_accept(port->priv, port->id, mac_bcast, true);
6632 if (err) {
6633 netdev_err(dev, "mvpp2_prs_mac_da_accept BC failed\n");
6634 return err;
6635 }
6636 err = mvpp2_prs_mac_da_accept(port->priv, port->id,
6637 dev->dev_addr, true);
6638 if (err) {
6639 netdev_err(dev, "mvpp2_prs_mac_da_accept MC failed\n");
6640 return err;
6641 }
6642 err = mvpp2_prs_tag_mode_set(port->priv, port->id, MVPP2_TAG_TYPE_MH);
6643 if (err) {
6644 netdev_err(dev, "mvpp2_prs_tag_mode_set failed\n");
6645 return err;
6646 }
6647 err = mvpp2_prs_def_flow(port);
6648 if (err) {
6649 netdev_err(dev, "mvpp2_prs_def_flow failed\n");
6650 return err;
6651 }
6652
6653 /* Allocate the Rx/Tx queues */
6654 err = mvpp2_setup_rxqs(port);
6655 if (err) {
6656 netdev_err(port->dev, "cannot allocate Rx queues\n");
6657 return err;
6658 }
6659
6660 err = mvpp2_setup_txqs(port);
6661 if (err) {
6662 netdev_err(port->dev, "cannot allocate Tx queues\n");
6663 goto err_cleanup_rxqs;
6664 }
6665
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006666 err = mvpp2_irqs_init(port);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006667 if (err) {
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006668 netdev_err(port->dev, "cannot init IRQs\n");
Marcin Wojtas3f518502014-07-10 16:52:13 -03006669 goto err_cleanup_txqs;
6670 }
6671
6672 /* In default link is down */
6673 netif_carrier_off(port->dev);
6674
6675 err = mvpp2_phy_connect(port);
6676 if (err < 0)
6677 goto err_free_irq;
6678
6679 /* Unmask interrupts on all CPUs */
6680 on_each_cpu(mvpp2_interrupts_unmask, port, 1);
Thomas Petazzoni213f4282017-08-03 10:42:00 +02006681 mvpp2_shared_interrupt_mask_unmask(port, false);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006682
6683 mvpp2_start_dev(port);
6684
6685 return 0;
6686
6687err_free_irq:
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006688 mvpp2_irqs_deinit(port);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006689err_cleanup_txqs:
6690 mvpp2_cleanup_txqs(port);
6691err_cleanup_rxqs:
6692 mvpp2_cleanup_rxqs(port);
6693 return err;
6694}
6695
6696static int mvpp2_stop(struct net_device *dev)
6697{
6698 struct mvpp2_port *port = netdev_priv(dev);
Marcin Wojtasedc660f2015-08-06 19:00:30 +02006699 struct mvpp2_port_pcpu *port_pcpu;
6700 int cpu;
Marcin Wojtas3f518502014-07-10 16:52:13 -03006701
6702 mvpp2_stop_dev(port);
6703 mvpp2_phy_disconnect(port);
6704
6705 /* Mask interrupts on all CPUs */
6706 on_each_cpu(mvpp2_interrupts_mask, port, 1);
Thomas Petazzoni213f4282017-08-03 10:42:00 +02006707 mvpp2_shared_interrupt_mask_unmask(port, true);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006708
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02006709 mvpp2_irqs_deinit(port);
Thomas Petazzoni213f4282017-08-03 10:42:00 +02006710 if (!port->has_tx_irqs) {
6711 for_each_present_cpu(cpu) {
6712 port_pcpu = per_cpu_ptr(port->pcpu, cpu);
Marcin Wojtasedc660f2015-08-06 19:00:30 +02006713
Thomas Petazzoni213f4282017-08-03 10:42:00 +02006714 hrtimer_cancel(&port_pcpu->tx_done_timer);
6715 port_pcpu->timer_scheduled = false;
6716 tasklet_kill(&port_pcpu->tx_done_tasklet);
6717 }
Marcin Wojtasedc660f2015-08-06 19:00:30 +02006718 }
Marcin Wojtas3f518502014-07-10 16:52:13 -03006719 mvpp2_cleanup_rxqs(port);
6720 mvpp2_cleanup_txqs(port);
6721
6722 return 0;
6723}
6724
6725static void mvpp2_set_rx_mode(struct net_device *dev)
6726{
6727 struct mvpp2_port *port = netdev_priv(dev);
6728 struct mvpp2 *priv = port->priv;
6729 struct netdev_hw_addr *ha;
6730 int id = port->id;
6731 bool allmulti = dev->flags & IFF_ALLMULTI;
6732
6733 mvpp2_prs_mac_promisc_set(priv, id, dev->flags & IFF_PROMISC);
6734 mvpp2_prs_mac_multi_set(priv, id, MVPP2_PE_MAC_MC_ALL, allmulti);
6735 mvpp2_prs_mac_multi_set(priv, id, MVPP2_PE_MAC_MC_IP6, allmulti);
6736
6737 /* Remove all port->id's mcast enries */
6738 mvpp2_prs_mcast_del_all(priv, id);
6739
6740 if (allmulti && !netdev_mc_empty(dev)) {
6741 netdev_for_each_mc_addr(ha, dev)
6742 mvpp2_prs_mac_da_accept(priv, id, ha->addr, true);
6743 }
6744}
6745
6746static int mvpp2_set_mac_address(struct net_device *dev, void *p)
6747{
6748 struct mvpp2_port *port = netdev_priv(dev);
6749 const struct sockaddr *addr = p;
6750 int err;
6751
6752 if (!is_valid_ether_addr(addr->sa_data)) {
6753 err = -EADDRNOTAVAIL;
Markus Elfringc1175542017-04-17 11:10:47 +02006754 goto log_error;
Marcin Wojtas3f518502014-07-10 16:52:13 -03006755 }
6756
6757 if (!netif_running(dev)) {
6758 err = mvpp2_prs_update_mac_da(dev, addr->sa_data);
6759 if (!err)
6760 return 0;
6761 /* Reconfigure parser to accept the original MAC address */
6762 err = mvpp2_prs_update_mac_da(dev, dev->dev_addr);
6763 if (err)
Markus Elfringc1175542017-04-17 11:10:47 +02006764 goto log_error;
Marcin Wojtas3f518502014-07-10 16:52:13 -03006765 }
6766
6767 mvpp2_stop_dev(port);
6768
6769 err = mvpp2_prs_update_mac_da(dev, addr->sa_data);
6770 if (!err)
6771 goto out_start;
6772
6773 /* Reconfigure parser accept the original MAC address */
6774 err = mvpp2_prs_update_mac_da(dev, dev->dev_addr);
6775 if (err)
Markus Elfringc1175542017-04-17 11:10:47 +02006776 goto log_error;
Marcin Wojtas3f518502014-07-10 16:52:13 -03006777out_start:
6778 mvpp2_start_dev(port);
6779 mvpp2_egress_enable(port);
6780 mvpp2_ingress_enable(port);
6781 return 0;
Markus Elfringc1175542017-04-17 11:10:47 +02006782log_error:
Markus Elfringdfd42402017-04-17 11:20:41 +02006783 netdev_err(dev, "failed to change MAC address\n");
Marcin Wojtas3f518502014-07-10 16:52:13 -03006784 return err;
6785}
6786
6787static int mvpp2_change_mtu(struct net_device *dev, int mtu)
6788{
6789 struct mvpp2_port *port = netdev_priv(dev);
6790 int err;
6791
Jarod Wilson57779872016-10-17 15:54:06 -04006792 if (!IS_ALIGNED(MVPP2_RX_PKT_SIZE(mtu), 8)) {
6793 netdev_info(dev, "illegal MTU value %d, round to %d\n", mtu,
6794 ALIGN(MVPP2_RX_PKT_SIZE(mtu), 8));
6795 mtu = ALIGN(MVPP2_RX_PKT_SIZE(mtu), 8);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006796 }
6797
6798 if (!netif_running(dev)) {
6799 err = mvpp2_bm_update_mtu(dev, mtu);
6800 if (!err) {
6801 port->pkt_size = MVPP2_RX_PKT_SIZE(mtu);
6802 return 0;
6803 }
6804
6805 /* Reconfigure BM to the original MTU */
6806 err = mvpp2_bm_update_mtu(dev, dev->mtu);
6807 if (err)
Markus Elfringc1175542017-04-17 11:10:47 +02006808 goto log_error;
Marcin Wojtas3f518502014-07-10 16:52:13 -03006809 }
6810
6811 mvpp2_stop_dev(port);
6812
6813 err = mvpp2_bm_update_mtu(dev, mtu);
6814 if (!err) {
6815 port->pkt_size = MVPP2_RX_PKT_SIZE(mtu);
6816 goto out_start;
6817 }
6818
6819 /* Reconfigure BM to the original MTU */
6820 err = mvpp2_bm_update_mtu(dev, dev->mtu);
6821 if (err)
Markus Elfringc1175542017-04-17 11:10:47 +02006822 goto log_error;
Marcin Wojtas3f518502014-07-10 16:52:13 -03006823
6824out_start:
6825 mvpp2_start_dev(port);
6826 mvpp2_egress_enable(port);
6827 mvpp2_ingress_enable(port);
6828
6829 return 0;
Markus Elfringc1175542017-04-17 11:10:47 +02006830log_error:
Markus Elfringdfd42402017-04-17 11:20:41 +02006831 netdev_err(dev, "failed to change MTU\n");
Marcin Wojtas3f518502014-07-10 16:52:13 -03006832 return err;
6833}
6834
stephen hemmingerbc1f4472017-01-06 19:12:52 -08006835static void
Marcin Wojtas3f518502014-07-10 16:52:13 -03006836mvpp2_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
6837{
6838 struct mvpp2_port *port = netdev_priv(dev);
6839 unsigned int start;
6840 int cpu;
6841
6842 for_each_possible_cpu(cpu) {
6843 struct mvpp2_pcpu_stats *cpu_stats;
6844 u64 rx_packets;
6845 u64 rx_bytes;
6846 u64 tx_packets;
6847 u64 tx_bytes;
6848
6849 cpu_stats = per_cpu_ptr(port->stats, cpu);
6850 do {
6851 start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
6852 rx_packets = cpu_stats->rx_packets;
6853 rx_bytes = cpu_stats->rx_bytes;
6854 tx_packets = cpu_stats->tx_packets;
6855 tx_bytes = cpu_stats->tx_bytes;
6856 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
6857
6858 stats->rx_packets += rx_packets;
6859 stats->rx_bytes += rx_bytes;
6860 stats->tx_packets += tx_packets;
6861 stats->tx_bytes += tx_bytes;
6862 }
6863
6864 stats->rx_errors = dev->stats.rx_errors;
6865 stats->rx_dropped = dev->stats.rx_dropped;
6866 stats->tx_dropped = dev->stats.tx_dropped;
Marcin Wojtas3f518502014-07-10 16:52:13 -03006867}
6868
Thomas Petazzonibd695a52014-07-27 23:21:36 +02006869static int mvpp2_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
6870{
Thomas Petazzonibd695a52014-07-27 23:21:36 +02006871 int ret;
6872
Philippe Reynes8e072692016-06-28 00:08:11 +02006873 if (!dev->phydev)
Thomas Petazzonibd695a52014-07-27 23:21:36 +02006874 return -ENOTSUPP;
6875
Philippe Reynes8e072692016-06-28 00:08:11 +02006876 ret = phy_mii_ioctl(dev->phydev, ifr, cmd);
Thomas Petazzonibd695a52014-07-27 23:21:36 +02006877 if (!ret)
6878 mvpp2_link_event(dev);
6879
6880 return ret;
6881}
6882
Marcin Wojtas3f518502014-07-10 16:52:13 -03006883/* Ethtool methods */
6884
Marcin Wojtas3f518502014-07-10 16:52:13 -03006885/* Set interrupt coalescing for ethtools */
6886static int mvpp2_ethtool_set_coalesce(struct net_device *dev,
6887 struct ethtool_coalesce *c)
6888{
6889 struct mvpp2_port *port = netdev_priv(dev);
6890 int queue;
6891
Thomas Petazzoni09f83972017-08-03 10:41:57 +02006892 for (queue = 0; queue < port->nrxqs; queue++) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03006893 struct mvpp2_rx_queue *rxq = port->rxqs[queue];
6894
6895 rxq->time_coal = c->rx_coalesce_usecs;
6896 rxq->pkts_coal = c->rx_max_coalesced_frames;
Thomas Petazzonid63f9e42017-02-21 11:28:02 +01006897 mvpp2_rx_pkts_coal_set(port, rxq);
6898 mvpp2_rx_time_coal_set(port, rxq);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006899 }
6900
Thomas Petazzoni213f4282017-08-03 10:42:00 +02006901 if (port->has_tx_irqs) {
6902 port->tx_time_coal = c->tx_coalesce_usecs;
6903 mvpp2_tx_time_coal_set(port);
6904 }
6905
Thomas Petazzoni09f83972017-08-03 10:41:57 +02006906 for (queue = 0; queue < port->ntxqs; queue++) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03006907 struct mvpp2_tx_queue *txq = port->txqs[queue];
6908
6909 txq->done_pkts_coal = c->tx_max_coalesced_frames;
Thomas Petazzoni213f4282017-08-03 10:42:00 +02006910
6911 if (port->has_tx_irqs)
6912 mvpp2_tx_pkts_coal_set(port, txq);
Marcin Wojtas3f518502014-07-10 16:52:13 -03006913 }
6914
Marcin Wojtas3f518502014-07-10 16:52:13 -03006915 return 0;
6916}
6917
6918/* get coalescing for ethtools */
6919static int mvpp2_ethtool_get_coalesce(struct net_device *dev,
6920 struct ethtool_coalesce *c)
6921{
6922 struct mvpp2_port *port = netdev_priv(dev);
6923
6924 c->rx_coalesce_usecs = port->rxqs[0]->time_coal;
6925 c->rx_max_coalesced_frames = port->rxqs[0]->pkts_coal;
6926 c->tx_max_coalesced_frames = port->txqs[0]->done_pkts_coal;
6927 return 0;
6928}
6929
6930static void mvpp2_ethtool_get_drvinfo(struct net_device *dev,
6931 struct ethtool_drvinfo *drvinfo)
6932{
6933 strlcpy(drvinfo->driver, MVPP2_DRIVER_NAME,
6934 sizeof(drvinfo->driver));
6935 strlcpy(drvinfo->version, MVPP2_DRIVER_VERSION,
6936 sizeof(drvinfo->version));
6937 strlcpy(drvinfo->bus_info, dev_name(&dev->dev),
6938 sizeof(drvinfo->bus_info));
6939}
6940
6941static void mvpp2_ethtool_get_ringparam(struct net_device *dev,
6942 struct ethtool_ringparam *ring)
6943{
6944 struct mvpp2_port *port = netdev_priv(dev);
6945
6946 ring->rx_max_pending = MVPP2_MAX_RXD;
6947 ring->tx_max_pending = MVPP2_MAX_TXD;
6948 ring->rx_pending = port->rx_ring_size;
6949 ring->tx_pending = port->tx_ring_size;
6950}
6951
6952static int mvpp2_ethtool_set_ringparam(struct net_device *dev,
6953 struct ethtool_ringparam *ring)
6954{
6955 struct mvpp2_port *port = netdev_priv(dev);
6956 u16 prev_rx_ring_size = port->rx_ring_size;
6957 u16 prev_tx_ring_size = port->tx_ring_size;
6958 int err;
6959
6960 err = mvpp2_check_ringparam_valid(dev, ring);
6961 if (err)
6962 return err;
6963
6964 if (!netif_running(dev)) {
6965 port->rx_ring_size = ring->rx_pending;
6966 port->tx_ring_size = ring->tx_pending;
6967 return 0;
6968 }
6969
6970 /* The interface is running, so we have to force a
6971 * reallocation of the queues
6972 */
6973 mvpp2_stop_dev(port);
6974 mvpp2_cleanup_rxqs(port);
6975 mvpp2_cleanup_txqs(port);
6976
6977 port->rx_ring_size = ring->rx_pending;
6978 port->tx_ring_size = ring->tx_pending;
6979
6980 err = mvpp2_setup_rxqs(port);
6981 if (err) {
6982 /* Reallocate Rx queues with the original ring size */
6983 port->rx_ring_size = prev_rx_ring_size;
6984 ring->rx_pending = prev_rx_ring_size;
6985 err = mvpp2_setup_rxqs(port);
6986 if (err)
6987 goto err_out;
6988 }
6989 err = mvpp2_setup_txqs(port);
6990 if (err) {
6991 /* Reallocate Tx queues with the original ring size */
6992 port->tx_ring_size = prev_tx_ring_size;
6993 ring->tx_pending = prev_tx_ring_size;
6994 err = mvpp2_setup_txqs(port);
6995 if (err)
6996 goto err_clean_rxqs;
6997 }
6998
6999 mvpp2_start_dev(port);
7000 mvpp2_egress_enable(port);
7001 mvpp2_ingress_enable(port);
7002
7003 return 0;
7004
7005err_clean_rxqs:
7006 mvpp2_cleanup_rxqs(port);
7007err_out:
Markus Elfringdfd42402017-04-17 11:20:41 +02007008 netdev_err(dev, "failed to change ring parameters");
Marcin Wojtas3f518502014-07-10 16:52:13 -03007009 return err;
7010}
7011
7012/* Device ops */
7013
7014static const struct net_device_ops mvpp2_netdev_ops = {
7015 .ndo_open = mvpp2_open,
7016 .ndo_stop = mvpp2_stop,
7017 .ndo_start_xmit = mvpp2_tx,
7018 .ndo_set_rx_mode = mvpp2_set_rx_mode,
7019 .ndo_set_mac_address = mvpp2_set_mac_address,
7020 .ndo_change_mtu = mvpp2_change_mtu,
7021 .ndo_get_stats64 = mvpp2_get_stats64,
Thomas Petazzonibd695a52014-07-27 23:21:36 +02007022 .ndo_do_ioctl = mvpp2_ioctl,
Marcin Wojtas3f518502014-07-10 16:52:13 -03007023};
7024
7025static const struct ethtool_ops mvpp2_eth_tool_ops = {
Florian Fainelli00606c42016-11-15 11:19:48 -08007026 .nway_reset = phy_ethtool_nway_reset,
Marcin Wojtas3f518502014-07-10 16:52:13 -03007027 .get_link = ethtool_op_get_link,
Marcin Wojtas3f518502014-07-10 16:52:13 -03007028 .set_coalesce = mvpp2_ethtool_set_coalesce,
7029 .get_coalesce = mvpp2_ethtool_get_coalesce,
7030 .get_drvinfo = mvpp2_ethtool_get_drvinfo,
7031 .get_ringparam = mvpp2_ethtool_get_ringparam,
7032 .set_ringparam = mvpp2_ethtool_set_ringparam,
Philippe Reynesfb773e92016-06-28 00:08:12 +02007033 .get_link_ksettings = phy_ethtool_get_link_ksettings,
7034 .set_link_ksettings = phy_ethtool_set_link_ksettings,
Marcin Wojtas3f518502014-07-10 16:52:13 -03007035};
7036
Thomas Petazzoni213f4282017-08-03 10:42:00 +02007037/* Used for PPv2.1, or PPv2.2 with the old Device Tree binding that
7038 * had a single IRQ defined per-port.
7039 */
7040static int mvpp2_simple_queue_vectors_init(struct mvpp2_port *port,
7041 struct device_node *port_node)
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02007042{
7043 struct mvpp2_queue_vector *v = &port->qvecs[0];
7044
7045 v->first_rxq = 0;
7046 v->nrxqs = port->nrxqs;
7047 v->type = MVPP2_QUEUE_VECTOR_SHARED;
7048 v->sw_thread_id = 0;
7049 v->sw_thread_mask = *cpumask_bits(cpu_online_mask);
7050 v->port = port;
7051 v->irq = irq_of_parse_and_map(port_node, 0);
7052 if (v->irq <= 0)
7053 return -EINVAL;
7054 netif_napi_add(port->dev, &v->napi, mvpp2_poll,
7055 NAPI_POLL_WEIGHT);
7056
7057 port->nqvecs = 1;
7058
7059 return 0;
7060}
7061
Thomas Petazzoni213f4282017-08-03 10:42:00 +02007062static int mvpp2_multi_queue_vectors_init(struct mvpp2_port *port,
7063 struct device_node *port_node)
7064{
7065 struct mvpp2_queue_vector *v;
7066 int i, ret;
7067
7068 port->nqvecs = num_possible_cpus();
7069 if (queue_mode == MVPP2_QDIST_SINGLE_MODE)
7070 port->nqvecs += 1;
7071
7072 for (i = 0; i < port->nqvecs; i++) {
7073 char irqname[16];
7074
7075 v = port->qvecs + i;
7076
7077 v->port = port;
7078 v->type = MVPP2_QUEUE_VECTOR_PRIVATE;
7079 v->sw_thread_id = i;
7080 v->sw_thread_mask = BIT(i);
7081
7082 snprintf(irqname, sizeof(irqname), "tx-cpu%d", i);
7083
7084 if (queue_mode == MVPP2_QDIST_MULTI_MODE) {
7085 v->first_rxq = i * MVPP2_DEFAULT_RXQ;
7086 v->nrxqs = MVPP2_DEFAULT_RXQ;
7087 } else if (queue_mode == MVPP2_QDIST_SINGLE_MODE &&
7088 i == (port->nqvecs - 1)) {
7089 v->first_rxq = 0;
7090 v->nrxqs = port->nrxqs;
7091 v->type = MVPP2_QUEUE_VECTOR_SHARED;
7092 strncpy(irqname, "rx-shared", sizeof(irqname));
7093 }
7094
7095 v->irq = of_irq_get_byname(port_node, irqname);
7096 if (v->irq <= 0) {
7097 ret = -EINVAL;
7098 goto err;
7099 }
7100
7101 netif_napi_add(port->dev, &v->napi, mvpp2_poll,
7102 NAPI_POLL_WEIGHT);
7103 }
7104
7105 return 0;
7106
7107err:
7108 for (i = 0; i < port->nqvecs; i++)
7109 irq_dispose_mapping(port->qvecs[i].irq);
7110 return ret;
7111}
7112
7113static int mvpp2_queue_vectors_init(struct mvpp2_port *port,
7114 struct device_node *port_node)
7115{
7116 if (port->has_tx_irqs)
7117 return mvpp2_multi_queue_vectors_init(port, port_node);
7118 else
7119 return mvpp2_simple_queue_vectors_init(port, port_node);
7120}
7121
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02007122static void mvpp2_queue_vectors_deinit(struct mvpp2_port *port)
7123{
7124 int i;
7125
7126 for (i = 0; i < port->nqvecs; i++)
7127 irq_dispose_mapping(port->qvecs[i].irq);
7128}
7129
7130/* Configure Rx queue group interrupt for this port */
7131static void mvpp2_rx_irqs_setup(struct mvpp2_port *port)
7132{
7133 struct mvpp2 *priv = port->priv;
7134 u32 val;
7135 int i;
7136
7137 if (priv->hw_version == MVPP21) {
7138 mvpp2_write(priv, MVPP21_ISR_RXQ_GROUP_REG(port->id),
7139 port->nrxqs);
7140 return;
7141 }
7142
7143 /* Handle the more complicated PPv2.2 case */
7144 for (i = 0; i < port->nqvecs; i++) {
7145 struct mvpp2_queue_vector *qv = port->qvecs + i;
7146
7147 if (!qv->nrxqs)
7148 continue;
7149
7150 val = qv->sw_thread_id;
7151 val |= port->id << MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_OFFSET;
7152 mvpp2_write(priv, MVPP22_ISR_RXQ_GROUP_INDEX_REG, val);
7153
7154 val = qv->first_rxq;
7155 val |= qv->nrxqs << MVPP22_ISR_RXQ_SUB_GROUP_SIZE_OFFSET;
7156 mvpp2_write(priv, MVPP22_ISR_RXQ_SUB_GROUP_CONFIG_REG, val);
7157 }
7158}
7159
Marcin Wojtas3f518502014-07-10 16:52:13 -03007160/* Initialize port HW */
7161static int mvpp2_port_init(struct mvpp2_port *port)
7162{
7163 struct device *dev = port->dev->dev.parent;
7164 struct mvpp2 *priv = port->priv;
7165 struct mvpp2_txq_pcpu *txq_pcpu;
7166 int queue, cpu, err;
7167
Thomas Petazzoni09f83972017-08-03 10:41:57 +02007168 /* Checks for hardware constraints */
7169 if (port->first_rxq + port->nrxqs >
Thomas Petazzoni59b9a312017-03-07 16:53:17 +01007170 MVPP2_MAX_PORTS * priv->max_port_rxqs)
Marcin Wojtas3f518502014-07-10 16:52:13 -03007171 return -EINVAL;
7172
Thomas Petazzoni09f83972017-08-03 10:41:57 +02007173 if (port->nrxqs % 4 || (port->nrxqs > priv->max_port_rxqs) ||
7174 (port->ntxqs > MVPP2_MAX_TXQ))
7175 return -EINVAL;
7176
Marcin Wojtas3f518502014-07-10 16:52:13 -03007177 /* Disable port */
7178 mvpp2_egress_disable(port);
7179 mvpp2_port_disable(port);
7180
Thomas Petazzoni213f4282017-08-03 10:42:00 +02007181 port->tx_time_coal = MVPP2_TXDONE_COAL_USEC;
7182
Thomas Petazzoni09f83972017-08-03 10:41:57 +02007183 port->txqs = devm_kcalloc(dev, port->ntxqs, sizeof(*port->txqs),
Marcin Wojtas3f518502014-07-10 16:52:13 -03007184 GFP_KERNEL);
7185 if (!port->txqs)
7186 return -ENOMEM;
7187
7188 /* Associate physical Tx queues to this port and initialize.
7189 * The mapping is predefined.
7190 */
Thomas Petazzoni09f83972017-08-03 10:41:57 +02007191 for (queue = 0; queue < port->ntxqs; queue++) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03007192 int queue_phy_id = mvpp2_txq_phys(port->id, queue);
7193 struct mvpp2_tx_queue *txq;
7194
7195 txq = devm_kzalloc(dev, sizeof(*txq), GFP_KERNEL);
Christophe Jaillet177c8d12017-02-19 10:19:57 +01007196 if (!txq) {
7197 err = -ENOMEM;
7198 goto err_free_percpu;
7199 }
Marcin Wojtas3f518502014-07-10 16:52:13 -03007200
7201 txq->pcpu = alloc_percpu(struct mvpp2_txq_pcpu);
7202 if (!txq->pcpu) {
7203 err = -ENOMEM;
7204 goto err_free_percpu;
7205 }
7206
7207 txq->id = queue_phy_id;
7208 txq->log_id = queue;
7209 txq->done_pkts_coal = MVPP2_TXDONE_COAL_PKTS_THRESH;
7210 for_each_present_cpu(cpu) {
7211 txq_pcpu = per_cpu_ptr(txq->pcpu, cpu);
7212 txq_pcpu->cpu = cpu;
7213 }
7214
7215 port->txqs[queue] = txq;
7216 }
7217
Thomas Petazzoni09f83972017-08-03 10:41:57 +02007218 port->rxqs = devm_kcalloc(dev, port->nrxqs, sizeof(*port->rxqs),
Marcin Wojtas3f518502014-07-10 16:52:13 -03007219 GFP_KERNEL);
7220 if (!port->rxqs) {
7221 err = -ENOMEM;
7222 goto err_free_percpu;
7223 }
7224
7225 /* Allocate and initialize Rx queue for this port */
Thomas Petazzoni09f83972017-08-03 10:41:57 +02007226 for (queue = 0; queue < port->nrxqs; queue++) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03007227 struct mvpp2_rx_queue *rxq;
7228
7229 /* Map physical Rx queue to port's logical Rx queue */
7230 rxq = devm_kzalloc(dev, sizeof(*rxq), GFP_KERNEL);
Jisheng Zhangd82b0c22016-03-31 17:01:23 +08007231 if (!rxq) {
7232 err = -ENOMEM;
Marcin Wojtas3f518502014-07-10 16:52:13 -03007233 goto err_free_percpu;
Jisheng Zhangd82b0c22016-03-31 17:01:23 +08007234 }
Marcin Wojtas3f518502014-07-10 16:52:13 -03007235 /* Map this Rx queue to a physical queue */
7236 rxq->id = port->first_rxq + queue;
7237 rxq->port = port->id;
7238 rxq->logic_rxq = queue;
7239
7240 port->rxqs[queue] = rxq;
7241 }
7242
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02007243 mvpp2_rx_irqs_setup(port);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007244
7245 /* Create Rx descriptor rings */
Thomas Petazzoni09f83972017-08-03 10:41:57 +02007246 for (queue = 0; queue < port->nrxqs; queue++) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03007247 struct mvpp2_rx_queue *rxq = port->rxqs[queue];
7248
7249 rxq->size = port->rx_ring_size;
7250 rxq->pkts_coal = MVPP2_RX_COAL_PKTS;
7251 rxq->time_coal = MVPP2_RX_COAL_USEC;
7252 }
7253
7254 mvpp2_ingress_disable(port);
7255
7256 /* Port default configuration */
7257 mvpp2_defaults_set(port);
7258
7259 /* Port's classifier configuration */
7260 mvpp2_cls_oversize_rxq_set(port);
7261 mvpp2_cls_port_config(port);
7262
7263 /* Provide an initial Rx packet size */
7264 port->pkt_size = MVPP2_RX_PKT_SIZE(port->dev->mtu);
7265
7266 /* Initialize pools for swf */
7267 err = mvpp2_swf_bm_pool_init(port);
7268 if (err)
7269 goto err_free_percpu;
7270
7271 return 0;
7272
7273err_free_percpu:
Thomas Petazzoni09f83972017-08-03 10:41:57 +02007274 for (queue = 0; queue < port->ntxqs; queue++) {
Marcin Wojtas3f518502014-07-10 16:52:13 -03007275 if (!port->txqs[queue])
7276 continue;
7277 free_percpu(port->txqs[queue]->pcpu);
7278 }
7279 return err;
7280}
7281
Thomas Petazzoni213f4282017-08-03 10:42:00 +02007282/* Checks if the port DT description has the TX interrupts
7283 * described. On PPv2.1, there are no such interrupts. On PPv2.2,
7284 * there are available, but we need to keep support for old DTs.
7285 */
7286static bool mvpp2_port_has_tx_irqs(struct mvpp2 *priv,
7287 struct device_node *port_node)
7288{
7289 char *irqs[5] = { "rx-shared", "tx-cpu0", "tx-cpu1",
7290 "tx-cpu2", "tx-cpu3" };
7291 int ret, i;
7292
7293 if (priv->hw_version == MVPP21)
7294 return false;
7295
7296 for (i = 0; i < 5; i++) {
7297 ret = of_property_match_string(port_node, "interrupt-names",
7298 irqs[i]);
7299 if (ret < 0)
7300 return false;
7301 }
7302
7303 return true;
7304}
7305
Marcin Wojtas3f518502014-07-10 16:52:13 -03007306/* Ports initialization */
7307static int mvpp2_port_probe(struct platform_device *pdev,
7308 struct device_node *port_node,
Thomas Petazzoni59b9a312017-03-07 16:53:17 +01007309 struct mvpp2 *priv)
Marcin Wojtas3f518502014-07-10 16:52:13 -03007310{
7311 struct device_node *phy_node;
Antoine Tenart542897d2017-08-30 10:29:15 +02007312 struct phy *comphy;
Marcin Wojtas3f518502014-07-10 16:52:13 -03007313 struct mvpp2_port *port;
Marcin Wojtasedc660f2015-08-06 19:00:30 +02007314 struct mvpp2_port_pcpu *port_pcpu;
Marcin Wojtas3f518502014-07-10 16:52:13 -03007315 struct net_device *dev;
7316 struct resource *res;
7317 const char *dt_mac_addr;
7318 const char *mac_from;
7319 char hw_mac_addr[ETH_ALEN];
Thomas Petazzoni09f83972017-08-03 10:41:57 +02007320 unsigned int ntxqs, nrxqs;
Thomas Petazzoni213f4282017-08-03 10:42:00 +02007321 bool has_tx_irqs;
Marcin Wojtas3f518502014-07-10 16:52:13 -03007322 u32 id;
7323 int features;
7324 int phy_mode;
Marcin Wojtasedc660f2015-08-06 19:00:30 +02007325 int err, i, cpu;
Marcin Wojtas3f518502014-07-10 16:52:13 -03007326
Thomas Petazzoni213f4282017-08-03 10:42:00 +02007327 has_tx_irqs = mvpp2_port_has_tx_irqs(priv, port_node);
7328
7329 if (!has_tx_irqs)
7330 queue_mode = MVPP2_QDIST_SINGLE_MODE;
7331
Thomas Petazzoni09f83972017-08-03 10:41:57 +02007332 ntxqs = MVPP2_MAX_TXQ;
Thomas Petazzoni213f4282017-08-03 10:42:00 +02007333 if (priv->hw_version == MVPP22 && queue_mode == MVPP2_QDIST_MULTI_MODE)
7334 nrxqs = MVPP2_DEFAULT_RXQ * num_possible_cpus();
7335 else
7336 nrxqs = MVPP2_DEFAULT_RXQ;
Thomas Petazzoni09f83972017-08-03 10:41:57 +02007337
7338 dev = alloc_etherdev_mqs(sizeof(*port), ntxqs, nrxqs);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007339 if (!dev)
7340 return -ENOMEM;
7341
7342 phy_node = of_parse_phandle(port_node, "phy", 0);
7343 if (!phy_node) {
7344 dev_err(&pdev->dev, "missing phy\n");
7345 err = -ENODEV;
7346 goto err_free_netdev;
7347 }
7348
7349 phy_mode = of_get_phy_mode(port_node);
7350 if (phy_mode < 0) {
7351 dev_err(&pdev->dev, "incorrect phy mode\n");
7352 err = phy_mode;
7353 goto err_free_netdev;
7354 }
7355
Antoine Tenart542897d2017-08-30 10:29:15 +02007356 comphy = devm_of_phy_get(&pdev->dev, port_node, NULL);
7357 if (IS_ERR(comphy)) {
7358 if (PTR_ERR(comphy) == -EPROBE_DEFER) {
7359 err = -EPROBE_DEFER;
7360 goto err_free_netdev;
7361 }
7362 comphy = NULL;
7363 }
7364
Marcin Wojtas3f518502014-07-10 16:52:13 -03007365 if (of_property_read_u32(port_node, "port-id", &id)) {
7366 err = -EINVAL;
7367 dev_err(&pdev->dev, "missing port-id value\n");
7368 goto err_free_netdev;
7369 }
7370
7371 dev->tx_queue_len = MVPP2_MAX_TXD;
7372 dev->watchdog_timeo = 5 * HZ;
7373 dev->netdev_ops = &mvpp2_netdev_ops;
7374 dev->ethtool_ops = &mvpp2_eth_tool_ops;
7375
7376 port = netdev_priv(dev);
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02007377 port->dev = dev;
Thomas Petazzoni09f83972017-08-03 10:41:57 +02007378 port->ntxqs = ntxqs;
7379 port->nrxqs = nrxqs;
Thomas Petazzoni213f4282017-08-03 10:42:00 +02007380 port->priv = priv;
7381 port->has_tx_irqs = has_tx_irqs;
Marcin Wojtas3f518502014-07-10 16:52:13 -03007382
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02007383 err = mvpp2_queue_vectors_init(port, port_node);
7384 if (err)
Marcin Wojtas3f518502014-07-10 16:52:13 -03007385 goto err_free_netdev;
Marcin Wojtas3f518502014-07-10 16:52:13 -03007386
7387 if (of_property_read_bool(port_node, "marvell,loopback"))
7388 port->flags |= MVPP2_F_LOOPBACK;
7389
Marcin Wojtas3f518502014-07-10 16:52:13 -03007390 port->id = id;
Thomas Petazzoni59b9a312017-03-07 16:53:17 +01007391 if (priv->hw_version == MVPP21)
Thomas Petazzoni09f83972017-08-03 10:41:57 +02007392 port->first_rxq = port->id * port->nrxqs;
Thomas Petazzoni59b9a312017-03-07 16:53:17 +01007393 else
7394 port->first_rxq = port->id * priv->max_port_rxqs;
7395
Marcin Wojtas3f518502014-07-10 16:52:13 -03007396 port->phy_node = phy_node;
7397 port->phy_interface = phy_mode;
Antoine Tenart542897d2017-08-30 10:29:15 +02007398 port->comphy = comphy;
Marcin Wojtas3f518502014-07-10 16:52:13 -03007399
Thomas Petazzonia7868412017-03-07 16:53:13 +01007400 if (priv->hw_version == MVPP21) {
7401 res = platform_get_resource(pdev, IORESOURCE_MEM, 2 + id);
7402 port->base = devm_ioremap_resource(&pdev->dev, res);
7403 if (IS_ERR(port->base)) {
7404 err = PTR_ERR(port->base);
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02007405 goto err_deinit_qvecs;
Thomas Petazzonia7868412017-03-07 16:53:13 +01007406 }
7407 } else {
7408 if (of_property_read_u32(port_node, "gop-port-id",
7409 &port->gop_id)) {
7410 err = -EINVAL;
7411 dev_err(&pdev->dev, "missing gop-port-id value\n");
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02007412 goto err_deinit_qvecs;
Thomas Petazzonia7868412017-03-07 16:53:13 +01007413 }
7414
7415 port->base = priv->iface_base + MVPP22_GMAC_BASE(port->gop_id);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007416 }
7417
7418 /* Alloc per-cpu stats */
7419 port->stats = netdev_alloc_pcpu_stats(struct mvpp2_pcpu_stats);
7420 if (!port->stats) {
7421 err = -ENOMEM;
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02007422 goto err_deinit_qvecs;
Marcin Wojtas3f518502014-07-10 16:52:13 -03007423 }
7424
7425 dt_mac_addr = of_get_mac_address(port_node);
7426 if (dt_mac_addr && is_valid_ether_addr(dt_mac_addr)) {
7427 mac_from = "device tree";
7428 ether_addr_copy(dev->dev_addr, dt_mac_addr);
7429 } else {
Thomas Petazzoni26975822017-03-07 16:53:14 +01007430 if (priv->hw_version == MVPP21)
7431 mvpp21_get_mac_address(port, hw_mac_addr);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007432 if (is_valid_ether_addr(hw_mac_addr)) {
7433 mac_from = "hardware";
7434 ether_addr_copy(dev->dev_addr, hw_mac_addr);
7435 } else {
7436 mac_from = "random";
7437 eth_hw_addr_random(dev);
7438 }
7439 }
7440
7441 port->tx_ring_size = MVPP2_MAX_TXD;
7442 port->rx_ring_size = MVPP2_MAX_RXD;
Marcin Wojtas3f518502014-07-10 16:52:13 -03007443 SET_NETDEV_DEV(dev, &pdev->dev);
7444
7445 err = mvpp2_port_init(port);
7446 if (err < 0) {
7447 dev_err(&pdev->dev, "failed to init port %d\n", id);
7448 goto err_free_stats;
7449 }
Thomas Petazzoni26975822017-03-07 16:53:14 +01007450
Thomas Petazzoni26975822017-03-07 16:53:14 +01007451 mvpp2_port_periodic_xon_disable(port);
7452
7453 if (priv->hw_version == MVPP21)
7454 mvpp2_port_fc_adv_enable(port);
7455
7456 mvpp2_port_reset(port);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007457
Marcin Wojtasedc660f2015-08-06 19:00:30 +02007458 port->pcpu = alloc_percpu(struct mvpp2_port_pcpu);
7459 if (!port->pcpu) {
7460 err = -ENOMEM;
7461 goto err_free_txq_pcpu;
7462 }
7463
Thomas Petazzoni213f4282017-08-03 10:42:00 +02007464 if (!port->has_tx_irqs) {
7465 for_each_present_cpu(cpu) {
7466 port_pcpu = per_cpu_ptr(port->pcpu, cpu);
Marcin Wojtasedc660f2015-08-06 19:00:30 +02007467
Thomas Petazzoni213f4282017-08-03 10:42:00 +02007468 hrtimer_init(&port_pcpu->tx_done_timer, CLOCK_MONOTONIC,
7469 HRTIMER_MODE_REL_PINNED);
7470 port_pcpu->tx_done_timer.function = mvpp2_hr_timer_cb;
7471 port_pcpu->timer_scheduled = false;
Marcin Wojtasedc660f2015-08-06 19:00:30 +02007472
Thomas Petazzoni213f4282017-08-03 10:42:00 +02007473 tasklet_init(&port_pcpu->tx_done_tasklet,
7474 mvpp2_tx_proc_cb,
7475 (unsigned long)dev);
7476 }
Marcin Wojtasedc660f2015-08-06 19:00:30 +02007477 }
7478
Antoine Ténart186cd4d2017-08-23 09:46:56 +02007479 features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
Marcin Wojtas3f518502014-07-10 16:52:13 -03007480 dev->features = features | NETIF_F_RXCSUM;
7481 dev->hw_features |= features | NETIF_F_RXCSUM | NETIF_F_GRO;
7482 dev->vlan_features |= features;
7483
Jarod Wilson57779872016-10-17 15:54:06 -04007484 /* MTU range: 68 - 9676 */
7485 dev->min_mtu = ETH_MIN_MTU;
7486 /* 9676 == 9700 - 20 and rounding to 8 */
7487 dev->max_mtu = 9676;
7488
Marcin Wojtas3f518502014-07-10 16:52:13 -03007489 err = register_netdev(dev);
7490 if (err < 0) {
7491 dev_err(&pdev->dev, "failed to register netdev\n");
Marcin Wojtasedc660f2015-08-06 19:00:30 +02007492 goto err_free_port_pcpu;
Marcin Wojtas3f518502014-07-10 16:52:13 -03007493 }
7494 netdev_info(dev, "Using %s mac address %pM\n", mac_from, dev->dev_addr);
7495
Marcin Wojtas3f518502014-07-10 16:52:13 -03007496 priv->port_list[id] = port;
7497 return 0;
7498
Marcin Wojtasedc660f2015-08-06 19:00:30 +02007499err_free_port_pcpu:
7500 free_percpu(port->pcpu);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007501err_free_txq_pcpu:
Thomas Petazzoni09f83972017-08-03 10:41:57 +02007502 for (i = 0; i < port->ntxqs; i++)
Marcin Wojtas3f518502014-07-10 16:52:13 -03007503 free_percpu(port->txqs[i]->pcpu);
7504err_free_stats:
7505 free_percpu(port->stats);
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02007506err_deinit_qvecs:
7507 mvpp2_queue_vectors_deinit(port);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007508err_free_netdev:
Peter Chenccb80392016-08-01 15:02:37 +08007509 of_node_put(phy_node);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007510 free_netdev(dev);
7511 return err;
7512}
7513
7514/* Ports removal routine */
7515static void mvpp2_port_remove(struct mvpp2_port *port)
7516{
7517 int i;
7518
7519 unregister_netdev(port->dev);
Peter Chenccb80392016-08-01 15:02:37 +08007520 of_node_put(port->phy_node);
Marcin Wojtasedc660f2015-08-06 19:00:30 +02007521 free_percpu(port->pcpu);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007522 free_percpu(port->stats);
Thomas Petazzoni09f83972017-08-03 10:41:57 +02007523 for (i = 0; i < port->ntxqs; i++)
Marcin Wojtas3f518502014-07-10 16:52:13 -03007524 free_percpu(port->txqs[i]->pcpu);
Thomas Petazzoni591f4cf2017-08-03 10:41:59 +02007525 mvpp2_queue_vectors_deinit(port);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007526 free_netdev(port->dev);
7527}
7528
7529/* Initialize decoding windows */
7530static void mvpp2_conf_mbus_windows(const struct mbus_dram_target_info *dram,
7531 struct mvpp2 *priv)
7532{
7533 u32 win_enable;
7534 int i;
7535
7536 for (i = 0; i < 6; i++) {
7537 mvpp2_write(priv, MVPP2_WIN_BASE(i), 0);
7538 mvpp2_write(priv, MVPP2_WIN_SIZE(i), 0);
7539
7540 if (i < 4)
7541 mvpp2_write(priv, MVPP2_WIN_REMAP(i), 0);
7542 }
7543
7544 win_enable = 0;
7545
7546 for (i = 0; i < dram->num_cs; i++) {
7547 const struct mbus_dram_window *cs = dram->cs + i;
7548
7549 mvpp2_write(priv, MVPP2_WIN_BASE(i),
7550 (cs->base & 0xffff0000) | (cs->mbus_attr << 8) |
7551 dram->mbus_dram_target_id);
7552
7553 mvpp2_write(priv, MVPP2_WIN_SIZE(i),
7554 (cs->size - 1) & 0xffff0000);
7555
7556 win_enable |= (1 << i);
7557 }
7558
7559 mvpp2_write(priv, MVPP2_BASE_ADDR_ENABLE, win_enable);
7560}
7561
7562/* Initialize Rx FIFO's */
7563static void mvpp2_rx_fifo_init(struct mvpp2 *priv)
7564{
7565 int port;
7566
7567 for (port = 0; port < MVPP2_MAX_PORTS; port++) {
7568 mvpp2_write(priv, MVPP2_RX_DATA_FIFO_SIZE_REG(port),
7569 MVPP2_RX_FIFO_PORT_DATA_SIZE);
7570 mvpp2_write(priv, MVPP2_RX_ATTR_FIFO_SIZE_REG(port),
7571 MVPP2_RX_FIFO_PORT_ATTR_SIZE);
7572 }
7573
7574 mvpp2_write(priv, MVPP2_RX_MIN_PKT_SIZE_REG,
7575 MVPP2_RX_FIFO_PORT_MIN_PKT);
7576 mvpp2_write(priv, MVPP2_RX_FIFO_INIT_REG, 0x1);
7577}
7578
Thomas Petazzoni6763ce32017-03-07 16:53:15 +01007579static void mvpp2_axi_init(struct mvpp2 *priv)
7580{
7581 u32 val, rdval, wrval;
7582
7583 mvpp2_write(priv, MVPP22_BM_ADDR_HIGH_RLS_REG, 0x0);
7584
7585 /* AXI Bridge Configuration */
7586
7587 rdval = MVPP22_AXI_CODE_CACHE_RD_CACHE
7588 << MVPP22_AXI_ATTR_CACHE_OFFS;
7589 rdval |= MVPP22_AXI_CODE_DOMAIN_OUTER_DOM
7590 << MVPP22_AXI_ATTR_DOMAIN_OFFS;
7591
7592 wrval = MVPP22_AXI_CODE_CACHE_WR_CACHE
7593 << MVPP22_AXI_ATTR_CACHE_OFFS;
7594 wrval |= MVPP22_AXI_CODE_DOMAIN_OUTER_DOM
7595 << MVPP22_AXI_ATTR_DOMAIN_OFFS;
7596
7597 /* BM */
7598 mvpp2_write(priv, MVPP22_AXI_BM_WR_ATTR_REG, wrval);
7599 mvpp2_write(priv, MVPP22_AXI_BM_RD_ATTR_REG, rdval);
7600
7601 /* Descriptors */
7602 mvpp2_write(priv, MVPP22_AXI_AGGRQ_DESCR_RD_ATTR_REG, rdval);
7603 mvpp2_write(priv, MVPP22_AXI_TXQ_DESCR_WR_ATTR_REG, wrval);
7604 mvpp2_write(priv, MVPP22_AXI_TXQ_DESCR_RD_ATTR_REG, rdval);
7605 mvpp2_write(priv, MVPP22_AXI_RXQ_DESCR_WR_ATTR_REG, wrval);
7606
7607 /* Buffer Data */
7608 mvpp2_write(priv, MVPP22_AXI_TX_DATA_RD_ATTR_REG, rdval);
7609 mvpp2_write(priv, MVPP22_AXI_RX_DATA_WR_ATTR_REG, wrval);
7610
7611 val = MVPP22_AXI_CODE_CACHE_NON_CACHE
7612 << MVPP22_AXI_CODE_CACHE_OFFS;
7613 val |= MVPP22_AXI_CODE_DOMAIN_SYSTEM
7614 << MVPP22_AXI_CODE_DOMAIN_OFFS;
7615 mvpp2_write(priv, MVPP22_AXI_RD_NORMAL_CODE_REG, val);
7616 mvpp2_write(priv, MVPP22_AXI_WR_NORMAL_CODE_REG, val);
7617
7618 val = MVPP22_AXI_CODE_CACHE_RD_CACHE
7619 << MVPP22_AXI_CODE_CACHE_OFFS;
7620 val |= MVPP22_AXI_CODE_DOMAIN_OUTER_DOM
7621 << MVPP22_AXI_CODE_DOMAIN_OFFS;
7622
7623 mvpp2_write(priv, MVPP22_AXI_RD_SNOOP_CODE_REG, val);
7624
7625 val = MVPP22_AXI_CODE_CACHE_WR_CACHE
7626 << MVPP22_AXI_CODE_CACHE_OFFS;
7627 val |= MVPP22_AXI_CODE_DOMAIN_OUTER_DOM
7628 << MVPP22_AXI_CODE_DOMAIN_OFFS;
7629
7630 mvpp2_write(priv, MVPP22_AXI_WR_SNOOP_CODE_REG, val);
7631}
7632
Marcin Wojtas3f518502014-07-10 16:52:13 -03007633/* Initialize network controller common part HW */
7634static int mvpp2_init(struct platform_device *pdev, struct mvpp2 *priv)
7635{
7636 const struct mbus_dram_target_info *dram_target_info;
7637 int err, i;
Marcin Wojtas08a23752014-07-21 13:48:12 -03007638 u32 val;
Marcin Wojtas3f518502014-07-10 16:52:13 -03007639
Marcin Wojtas3f518502014-07-10 16:52:13 -03007640 /* MBUS windows configuration */
7641 dram_target_info = mv_mbus_dram_info();
7642 if (dram_target_info)
7643 mvpp2_conf_mbus_windows(dram_target_info, priv);
7644
Thomas Petazzoni6763ce32017-03-07 16:53:15 +01007645 if (priv->hw_version == MVPP22)
7646 mvpp2_axi_init(priv);
7647
Marcin Wojtas08a23752014-07-21 13:48:12 -03007648 /* Disable HW PHY polling */
Thomas Petazzoni26975822017-03-07 16:53:14 +01007649 if (priv->hw_version == MVPP21) {
7650 val = readl(priv->lms_base + MVPP2_PHY_AN_CFG0_REG);
7651 val |= MVPP2_PHY_AN_STOP_SMI0_MASK;
7652 writel(val, priv->lms_base + MVPP2_PHY_AN_CFG0_REG);
7653 } else {
7654 val = readl(priv->iface_base + MVPP22_SMI_MISC_CFG_REG);
7655 val &= ~MVPP22_SMI_POLLING_EN;
7656 writel(val, priv->iface_base + MVPP22_SMI_MISC_CFG_REG);
7657 }
Marcin Wojtas08a23752014-07-21 13:48:12 -03007658
Marcin Wojtas3f518502014-07-10 16:52:13 -03007659 /* Allocate and initialize aggregated TXQs */
7660 priv->aggr_txqs = devm_kcalloc(&pdev->dev, num_present_cpus(),
Markus Elfringd7ce3ce2017-04-17 08:48:23 +02007661 sizeof(*priv->aggr_txqs),
Marcin Wojtas3f518502014-07-10 16:52:13 -03007662 GFP_KERNEL);
7663 if (!priv->aggr_txqs)
7664 return -ENOMEM;
7665
7666 for_each_present_cpu(i) {
7667 priv->aggr_txqs[i].id = i;
7668 priv->aggr_txqs[i].size = MVPP2_AGGR_TXQ_SIZE;
Antoine Ténart85affd72017-08-23 09:46:55 +02007669 err = mvpp2_aggr_txq_init(pdev, &priv->aggr_txqs[i], i, priv);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007670 if (err < 0)
7671 return err;
7672 }
7673
7674 /* Rx Fifo Init */
7675 mvpp2_rx_fifo_init(priv);
7676
Thomas Petazzoni26975822017-03-07 16:53:14 +01007677 if (priv->hw_version == MVPP21)
7678 writel(MVPP2_EXT_GLOBAL_CTRL_DEFAULT,
7679 priv->lms_base + MVPP2_MNG_EXTENDED_GLOBAL_CTRL_REG);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007680
7681 /* Allow cache snoop when transmiting packets */
7682 mvpp2_write(priv, MVPP2_TX_SNOOP_REG, 0x1);
7683
7684 /* Buffer Manager initialization */
7685 err = mvpp2_bm_init(pdev, priv);
7686 if (err < 0)
7687 return err;
7688
7689 /* Parser default initialization */
7690 err = mvpp2_prs_default_init(pdev, priv);
7691 if (err < 0)
7692 return err;
7693
7694 /* Classifier default initialization */
7695 mvpp2_cls_init(priv);
7696
7697 return 0;
7698}
7699
7700static int mvpp2_probe(struct platform_device *pdev)
7701{
7702 struct device_node *dn = pdev->dev.of_node;
7703 struct device_node *port_node;
7704 struct mvpp2 *priv;
7705 struct resource *res;
Thomas Petazzonia7868412017-03-07 16:53:13 +01007706 void __iomem *base;
Thomas Petazzonidf089aa2017-08-03 10:41:58 +02007707 int port_count, i;
Marcin Wojtas3f518502014-07-10 16:52:13 -03007708 int err;
7709
Markus Elfring0b92e592017-04-17 08:38:32 +02007710 priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007711 if (!priv)
7712 return -ENOMEM;
7713
Thomas Petazzonifaca9242017-03-07 16:53:06 +01007714 priv->hw_version =
7715 (unsigned long)of_device_get_match_data(&pdev->dev);
7716
Marcin Wojtas3f518502014-07-10 16:52:13 -03007717 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
Thomas Petazzonia7868412017-03-07 16:53:13 +01007718 base = devm_ioremap_resource(&pdev->dev, res);
7719 if (IS_ERR(base))
7720 return PTR_ERR(base);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007721
Thomas Petazzonia7868412017-03-07 16:53:13 +01007722 if (priv->hw_version == MVPP21) {
7723 res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
7724 priv->lms_base = devm_ioremap_resource(&pdev->dev, res);
7725 if (IS_ERR(priv->lms_base))
7726 return PTR_ERR(priv->lms_base);
7727 } else {
7728 res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
7729 priv->iface_base = devm_ioremap_resource(&pdev->dev, res);
7730 if (IS_ERR(priv->iface_base))
7731 return PTR_ERR(priv->iface_base);
Antoine Ténartf84bf382017-08-22 19:08:27 +02007732
7733 priv->sysctrl_base =
7734 syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
7735 "marvell,system-controller");
7736 if (IS_ERR(priv->sysctrl_base))
7737 /* The system controller regmap is optional for dt
7738 * compatibility reasons. When not provided, the
7739 * configuration of the GoP relies on the
7740 * firmware/bootloader.
7741 */
7742 priv->sysctrl_base = NULL;
Thomas Petazzonia7868412017-03-07 16:53:13 +01007743 }
7744
Thomas Petazzonidf089aa2017-08-03 10:41:58 +02007745 for (i = 0; i < MVPP2_MAX_THREADS; i++) {
Thomas Petazzonia7868412017-03-07 16:53:13 +01007746 u32 addr_space_sz;
7747
7748 addr_space_sz = (priv->hw_version == MVPP21 ?
7749 MVPP21_ADDR_SPACE_SZ : MVPP22_ADDR_SPACE_SZ);
Thomas Petazzonidf089aa2017-08-03 10:41:58 +02007750 priv->swth_base[i] = base + i * addr_space_sz;
Thomas Petazzonia7868412017-03-07 16:53:13 +01007751 }
Marcin Wojtas3f518502014-07-10 16:52:13 -03007752
Thomas Petazzoni59b9a312017-03-07 16:53:17 +01007753 if (priv->hw_version == MVPP21)
7754 priv->max_port_rxqs = 8;
7755 else
7756 priv->max_port_rxqs = 32;
7757
Marcin Wojtas3f518502014-07-10 16:52:13 -03007758 priv->pp_clk = devm_clk_get(&pdev->dev, "pp_clk");
7759 if (IS_ERR(priv->pp_clk))
7760 return PTR_ERR(priv->pp_clk);
7761 err = clk_prepare_enable(priv->pp_clk);
7762 if (err < 0)
7763 return err;
7764
7765 priv->gop_clk = devm_clk_get(&pdev->dev, "gop_clk");
7766 if (IS_ERR(priv->gop_clk)) {
7767 err = PTR_ERR(priv->gop_clk);
7768 goto err_pp_clk;
7769 }
7770 err = clk_prepare_enable(priv->gop_clk);
7771 if (err < 0)
7772 goto err_pp_clk;
7773
Thomas Petazzonifceb55d2017-03-07 16:53:18 +01007774 if (priv->hw_version == MVPP22) {
7775 priv->mg_clk = devm_clk_get(&pdev->dev, "mg_clk");
7776 if (IS_ERR(priv->mg_clk)) {
7777 err = PTR_ERR(priv->mg_clk);
7778 goto err_gop_clk;
7779 }
7780
7781 err = clk_prepare_enable(priv->mg_clk);
7782 if (err < 0)
7783 goto err_gop_clk;
7784 }
7785
Marcin Wojtas3f518502014-07-10 16:52:13 -03007786 /* Get system's tclk rate */
7787 priv->tclk = clk_get_rate(priv->pp_clk);
7788
Thomas Petazzoni2067e0a2017-03-07 16:53:19 +01007789 if (priv->hw_version == MVPP22) {
7790 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(40));
7791 if (err)
7792 goto err_mg_clk;
7793 /* Sadly, the BM pools all share the same register to
7794 * store the high 32 bits of their address. So they
7795 * must all have the same high 32 bits, which forces
7796 * us to restrict coherent memory to DMA_BIT_MASK(32).
7797 */
7798 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
7799 if (err)
7800 goto err_mg_clk;
7801 }
7802
Marcin Wojtas3f518502014-07-10 16:52:13 -03007803 /* Initialize network controller */
7804 err = mvpp2_init(pdev, priv);
7805 if (err < 0) {
7806 dev_err(&pdev->dev, "failed to initialize controller\n");
Thomas Petazzonifceb55d2017-03-07 16:53:18 +01007807 goto err_mg_clk;
Marcin Wojtas3f518502014-07-10 16:52:13 -03007808 }
7809
7810 port_count = of_get_available_child_count(dn);
7811 if (port_count == 0) {
7812 dev_err(&pdev->dev, "no ports enabled\n");
Wei Yongjun575a1932014-07-20 22:02:43 +08007813 err = -ENODEV;
Thomas Petazzonifceb55d2017-03-07 16:53:18 +01007814 goto err_mg_clk;
Marcin Wojtas3f518502014-07-10 16:52:13 -03007815 }
7816
7817 priv->port_list = devm_kcalloc(&pdev->dev, port_count,
Markus Elfring0b92e592017-04-17 08:38:32 +02007818 sizeof(*priv->port_list),
7819 GFP_KERNEL);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007820 if (!priv->port_list) {
7821 err = -ENOMEM;
Thomas Petazzonifceb55d2017-03-07 16:53:18 +01007822 goto err_mg_clk;
Marcin Wojtas3f518502014-07-10 16:52:13 -03007823 }
7824
7825 /* Initialize ports */
Marcin Wojtas3f518502014-07-10 16:52:13 -03007826 for_each_available_child_of_node(dn, port_node) {
Thomas Petazzoni59b9a312017-03-07 16:53:17 +01007827 err = mvpp2_port_probe(pdev, port_node, priv);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007828 if (err < 0)
Thomas Petazzonifceb55d2017-03-07 16:53:18 +01007829 goto err_mg_clk;
Marcin Wojtas3f518502014-07-10 16:52:13 -03007830 }
7831
7832 platform_set_drvdata(pdev, priv);
7833 return 0;
7834
Thomas Petazzonifceb55d2017-03-07 16:53:18 +01007835err_mg_clk:
7836 if (priv->hw_version == MVPP22)
7837 clk_disable_unprepare(priv->mg_clk);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007838err_gop_clk:
7839 clk_disable_unprepare(priv->gop_clk);
7840err_pp_clk:
7841 clk_disable_unprepare(priv->pp_clk);
7842 return err;
7843}
7844
7845static int mvpp2_remove(struct platform_device *pdev)
7846{
7847 struct mvpp2 *priv = platform_get_drvdata(pdev);
7848 struct device_node *dn = pdev->dev.of_node;
7849 struct device_node *port_node;
7850 int i = 0;
7851
7852 for_each_available_child_of_node(dn, port_node) {
7853 if (priv->port_list[i])
7854 mvpp2_port_remove(priv->port_list[i]);
7855 i++;
7856 }
7857
7858 for (i = 0; i < MVPP2_BM_POOLS_NUM; i++) {
7859 struct mvpp2_bm_pool *bm_pool = &priv->bm_pools[i];
7860
7861 mvpp2_bm_pool_destroy(pdev, priv, bm_pool);
7862 }
7863
7864 for_each_present_cpu(i) {
7865 struct mvpp2_tx_queue *aggr_txq = &priv->aggr_txqs[i];
7866
7867 dma_free_coherent(&pdev->dev,
7868 MVPP2_AGGR_TXQ_SIZE * MVPP2_DESC_ALIGNED_SIZE,
7869 aggr_txq->descs,
Thomas Petazzoni20396132017-03-07 16:53:00 +01007870 aggr_txq->descs_dma);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007871 }
7872
Thomas Petazzonifceb55d2017-03-07 16:53:18 +01007873 clk_disable_unprepare(priv->mg_clk);
Marcin Wojtas3f518502014-07-10 16:52:13 -03007874 clk_disable_unprepare(priv->pp_clk);
7875 clk_disable_unprepare(priv->gop_clk);
7876
7877 return 0;
7878}
7879
7880static const struct of_device_id mvpp2_match[] = {
Thomas Petazzonifaca9242017-03-07 16:53:06 +01007881 {
7882 .compatible = "marvell,armada-375-pp2",
7883 .data = (void *)MVPP21,
7884 },
Thomas Petazzonifc5e1552017-03-07 16:53:20 +01007885 {
7886 .compatible = "marvell,armada-7k-pp22",
7887 .data = (void *)MVPP22,
7888 },
Marcin Wojtas3f518502014-07-10 16:52:13 -03007889 { }
7890};
7891MODULE_DEVICE_TABLE(of, mvpp2_match);
7892
7893static struct platform_driver mvpp2_driver = {
7894 .probe = mvpp2_probe,
7895 .remove = mvpp2_remove,
7896 .driver = {
7897 .name = MVPP2_DRIVER_NAME,
7898 .of_match_table = mvpp2_match,
7899 },
7900};
7901
7902module_platform_driver(mvpp2_driver);
7903
7904MODULE_DESCRIPTION("Marvell PPv2 Ethernet Driver - www.marvell.com");
7905MODULE_AUTHOR("Marcin Wojtas <mw@semihalf.com>");
Ezequiel Garciac6340992014-07-14 10:34:47 -03007906MODULE_LICENSE("GPL v2");