Version:  2.0.40 2.2.26 2.4.37 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9 4.10

Linux/drivers/net/ethernet/marvell/mvneta.c

  1 /*
  2  * Driver for Marvell NETA network card for Armada XP and Armada 370 SoCs.
  3  *
  4  * Copyright (C) 2012 Marvell
  5  *
  6  * Rami Rosen <rosenr@marvell.com>
  7  * Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
  8  *
  9  * This file is licensed under the terms of the GNU General Public
 10  * License version 2. This program is licensed "as is" without any
 11  * warranty of any kind, whether express or implied.
 12  */
 13 
 14 #include <linux/clk.h>
 15 #include <linux/cpu.h>
 16 #include <linux/etherdevice.h>
 17 #include <linux/if_vlan.h>
 18 #include <linux/inetdevice.h>
 19 #include <linux/interrupt.h>
 20 #include <linux/io.h>
 21 #include <linux/kernel.h>
 22 #include <linux/mbus.h>
 23 #include <linux/module.h>
 24 #include <linux/netdevice.h>
 25 #include <linux/of.h>
 26 #include <linux/of_address.h>
 27 #include <linux/of_irq.h>
 28 #include <linux/of_mdio.h>
 29 #include <linux/of_net.h>
 30 #include <linux/phy.h>
 31 #include <linux/platform_device.h>
 32 #include <linux/skbuff.h>
 33 #include <net/hwbm.h>
 34 #include "mvneta_bm.h"
 35 #include <net/ip.h>
 36 #include <net/ipv6.h>
 37 #include <net/tso.h>
 38 
 39 /* Registers */
 40 #define MVNETA_RXQ_CONFIG_REG(q)                (0x1400 + ((q) << 2))
 41 #define      MVNETA_RXQ_HW_BUF_ALLOC            BIT(0)
 42 #define      MVNETA_RXQ_SHORT_POOL_ID_SHIFT     4
 43 #define      MVNETA_RXQ_SHORT_POOL_ID_MASK      0x30
 44 #define      MVNETA_RXQ_LONG_POOL_ID_SHIFT      6
 45 #define      MVNETA_RXQ_LONG_POOL_ID_MASK       0xc0
 46 #define      MVNETA_RXQ_PKT_OFFSET_ALL_MASK     (0xf    << 8)
 47 #define      MVNETA_RXQ_PKT_OFFSET_MASK(offs)   ((offs) << 8)
 48 #define MVNETA_RXQ_THRESHOLD_REG(q)             (0x14c0 + ((q) << 2))
 49 #define      MVNETA_RXQ_NON_OCCUPIED(v)         ((v) << 16)
 50 #define MVNETA_RXQ_BASE_ADDR_REG(q)             (0x1480 + ((q) << 2))
 51 #define MVNETA_RXQ_SIZE_REG(q)                  (0x14a0 + ((q) << 2))
 52 #define      MVNETA_RXQ_BUF_SIZE_SHIFT          19
 53 #define      MVNETA_RXQ_BUF_SIZE_MASK           (0x1fff << 19)
 54 #define MVNETA_RXQ_STATUS_REG(q)                (0x14e0 + ((q) << 2))
 55 #define      MVNETA_RXQ_OCCUPIED_ALL_MASK       0x3fff
 56 #define MVNETA_RXQ_STATUS_UPDATE_REG(q)         (0x1500 + ((q) << 2))
 57 #define      MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT  16
 58 #define      MVNETA_RXQ_ADD_NON_OCCUPIED_MAX    255
 59 #define MVNETA_PORT_POOL_BUFFER_SZ_REG(pool)    (0x1700 + ((pool) << 2))
 60 #define      MVNETA_PORT_POOL_BUFFER_SZ_SHIFT   3
 61 #define      MVNETA_PORT_POOL_BUFFER_SZ_MASK    0xfff8
 62 #define MVNETA_PORT_RX_RESET                    0x1cc0
 63 #define      MVNETA_PORT_RX_DMA_RESET           BIT(0)
 64 #define MVNETA_PHY_ADDR                         0x2000
 65 #define      MVNETA_PHY_ADDR_MASK               0x1f
 66 #define MVNETA_MBUS_RETRY                       0x2010
 67 #define MVNETA_UNIT_INTR_CAUSE                  0x2080
 68 #define MVNETA_UNIT_CONTROL                     0x20B0
 69 #define      MVNETA_PHY_POLLING_ENABLE          BIT(1)
 70 #define MVNETA_WIN_BASE(w)                      (0x2200 + ((w) << 3))
 71 #define MVNETA_WIN_SIZE(w)                      (0x2204 + ((w) << 3))
 72 #define MVNETA_WIN_REMAP(w)                     (0x2280 + ((w) << 2))
 73 #define MVNETA_BASE_ADDR_ENABLE                 0x2290
 74 #define MVNETA_ACCESS_PROTECT_ENABLE            0x2294
 75 #define MVNETA_PORT_CONFIG                      0x2400
 76 #define      MVNETA_UNI_PROMISC_MODE            BIT(0)
 77 #define      MVNETA_DEF_RXQ(q)                  ((q) << 1)
 78 #define      MVNETA_DEF_RXQ_ARP(q)              ((q) << 4)
 79 #define      MVNETA_TX_UNSET_ERR_SUM            BIT(12)
 80 #define      MVNETA_DEF_RXQ_TCP(q)              ((q) << 16)
 81 #define      MVNETA_DEF_RXQ_UDP(q)              ((q) << 19)
 82 #define      MVNETA_DEF_RXQ_BPDU(q)             ((q) << 22)
 83 #define      MVNETA_RX_CSUM_WITH_PSEUDO_HDR     BIT(25)
 84 #define      MVNETA_PORT_CONFIG_DEFL_VALUE(q)   (MVNETA_DEF_RXQ(q)       | \
 85                                                  MVNETA_DEF_RXQ_ARP(q)   | \
 86                                                  MVNETA_DEF_RXQ_TCP(q)   | \
 87                                                  MVNETA_DEF_RXQ_UDP(q)   | \
 88                                                  MVNETA_DEF_RXQ_BPDU(q)  | \
 89                                                  MVNETA_TX_UNSET_ERR_SUM | \
 90                                                  MVNETA_RX_CSUM_WITH_PSEUDO_HDR)
 91 #define MVNETA_PORT_CONFIG_EXTEND                0x2404
 92 #define MVNETA_MAC_ADDR_LOW                      0x2414
 93 #define MVNETA_MAC_ADDR_HIGH                     0x2418
 94 #define MVNETA_SDMA_CONFIG                       0x241c
 95 #define      MVNETA_SDMA_BRST_SIZE_16            4
 96 #define      MVNETA_RX_BRST_SZ_MASK(burst)       ((burst) << 1)
 97 #define      MVNETA_RX_NO_DATA_SWAP              BIT(4)
 98 #define      MVNETA_TX_NO_DATA_SWAP              BIT(5)
 99 #define      MVNETA_DESC_SWAP                    BIT(6)
100 #define      MVNETA_TX_BRST_SZ_MASK(burst)       ((burst) << 22)
101 #define MVNETA_PORT_STATUS                       0x2444
102 #define      MVNETA_TX_IN_PRGRS                  BIT(1)
103 #define      MVNETA_TX_FIFO_EMPTY                BIT(8)
104 #define MVNETA_RX_MIN_FRAME_SIZE                 0x247c
105 #define MVNETA_SERDES_CFG                        0x24A0
106 #define      MVNETA_SGMII_SERDES_PROTO           0x0cc7
107 #define      MVNETA_QSGMII_SERDES_PROTO          0x0667
108 #define MVNETA_TYPE_PRIO                         0x24bc
109 #define      MVNETA_FORCE_UNI                    BIT(21)
110 #define MVNETA_TXQ_CMD_1                         0x24e4
111 #define MVNETA_TXQ_CMD                           0x2448
112 #define      MVNETA_TXQ_DISABLE_SHIFT            8
113 #define      MVNETA_TXQ_ENABLE_MASK              0x000000ff
114 #define MVNETA_RX_DISCARD_FRAME_COUNT            0x2484
115 #define MVNETA_OVERRUN_FRAME_COUNT               0x2488
116 #define MVNETA_GMAC_CLOCK_DIVIDER                0x24f4
117 #define      MVNETA_GMAC_1MS_CLOCK_ENABLE        BIT(31)
118 #define MVNETA_ACC_MODE                          0x2500
119 #define MVNETA_BM_ADDRESS                        0x2504
120 #define MVNETA_CPU_MAP(cpu)                      (0x2540 + ((cpu) << 2))
121 #define      MVNETA_CPU_RXQ_ACCESS_ALL_MASK      0x000000ff
122 #define      MVNETA_CPU_TXQ_ACCESS_ALL_MASK      0x0000ff00
123 #define      MVNETA_CPU_RXQ_ACCESS(rxq)          BIT(rxq)
124 #define      MVNETA_CPU_TXQ_ACCESS(txq)          BIT(txq + 8)
125 #define MVNETA_RXQ_TIME_COAL_REG(q)              (0x2580 + ((q) << 2))
126 
127 /* Exception Interrupt Port/Queue Cause register
128  *
129  * Their behavior depend of the mapping done using the PCPX2Q
130  * registers. For a given CPU if the bit associated to a queue is not
131  * set, then for the register a read from this CPU will always return
132  * 0 and a write won't do anything
133  */
134 
135 #define MVNETA_INTR_NEW_CAUSE                    0x25a0
136 #define MVNETA_INTR_NEW_MASK                     0x25a4
137 
138 /* bits  0..7  = TXQ SENT, one bit per queue.
139  * bits  8..15 = RXQ OCCUP, one bit per queue.
140  * bits 16..23 = RXQ FREE, one bit per queue.
141  * bit  29 = OLD_REG_SUM, see old reg ?
142  * bit  30 = TX_ERR_SUM, one bit for 4 ports
143  * bit  31 = MISC_SUM,   one bit for 4 ports
144  */
145 #define      MVNETA_TX_INTR_MASK(nr_txqs)        (((1 << nr_txqs) - 1) << 0)
146 #define      MVNETA_TX_INTR_MASK_ALL             (0xff << 0)
147 #define      MVNETA_RX_INTR_MASK(nr_rxqs)        (((1 << nr_rxqs) - 1) << 8)
148 #define      MVNETA_RX_INTR_MASK_ALL             (0xff << 8)
149 #define      MVNETA_MISCINTR_INTR_MASK           BIT(31)
150 
151 #define MVNETA_INTR_OLD_CAUSE                    0x25a8
152 #define MVNETA_INTR_OLD_MASK                     0x25ac
153 
154 /* Data Path Port/Queue Cause Register */
155 #define MVNETA_INTR_MISC_CAUSE                   0x25b0
156 #define MVNETA_INTR_MISC_MASK                    0x25b4
157 
158 #define      MVNETA_CAUSE_PHY_STATUS_CHANGE      BIT(0)
159 #define      MVNETA_CAUSE_LINK_CHANGE            BIT(1)
160 #define      MVNETA_CAUSE_PTP                    BIT(4)
161 
162 #define      MVNETA_CAUSE_INTERNAL_ADDR_ERR      BIT(7)
163 #define      MVNETA_CAUSE_RX_OVERRUN             BIT(8)
164 #define      MVNETA_CAUSE_RX_CRC_ERROR           BIT(9)
165 #define      MVNETA_CAUSE_RX_LARGE_PKT           BIT(10)
166 #define      MVNETA_CAUSE_TX_UNDERUN             BIT(11)
167 #define      MVNETA_CAUSE_PRBS_ERR               BIT(12)
168 #define      MVNETA_CAUSE_PSC_SYNC_CHANGE        BIT(13)
169 #define      MVNETA_CAUSE_SERDES_SYNC_ERR        BIT(14)
170 
171 #define      MVNETA_CAUSE_BMU_ALLOC_ERR_SHIFT    16
172 #define      MVNETA_CAUSE_BMU_ALLOC_ERR_ALL_MASK   (0xF << MVNETA_CAUSE_BMU_ALLOC_ERR_SHIFT)
173 #define      MVNETA_CAUSE_BMU_ALLOC_ERR_MASK(pool) (1 << (MVNETA_CAUSE_BMU_ALLOC_ERR_SHIFT + (pool)))
174 
175 #define      MVNETA_CAUSE_TXQ_ERROR_SHIFT        24
176 #define      MVNETA_CAUSE_TXQ_ERROR_ALL_MASK     (0xFF << MVNETA_CAUSE_TXQ_ERROR_SHIFT)
177 #define      MVNETA_CAUSE_TXQ_ERROR_MASK(q)      (1 << (MVNETA_CAUSE_TXQ_ERROR_SHIFT + (q)))
178 
179 #define MVNETA_INTR_ENABLE                       0x25b8
180 #define      MVNETA_TXQ_INTR_ENABLE_ALL_MASK     0x0000ff00
181 #define      MVNETA_RXQ_INTR_ENABLE_ALL_MASK     0x000000ff
182 
183 #define MVNETA_RXQ_CMD                           0x2680
184 #define      MVNETA_RXQ_DISABLE_SHIFT            8
185 #define      MVNETA_RXQ_ENABLE_MASK              0x000000ff
186 #define MVETH_TXQ_TOKEN_COUNT_REG(q)             (0x2700 + ((q) << 4))
187 #define MVETH_TXQ_TOKEN_CFG_REG(q)               (0x2704 + ((q) << 4))
188 #define MVNETA_GMAC_CTRL_0                       0x2c00
189 #define      MVNETA_GMAC_MAX_RX_SIZE_SHIFT       2
190 #define      MVNETA_GMAC_MAX_RX_SIZE_MASK        0x7ffc
191 #define      MVNETA_GMAC0_PORT_ENABLE            BIT(0)
192 #define MVNETA_GMAC_CTRL_2                       0x2c08
193 #define      MVNETA_GMAC2_INBAND_AN_ENABLE       BIT(0)
194 #define      MVNETA_GMAC2_PCS_ENABLE             BIT(3)
195 #define      MVNETA_GMAC2_PORT_RGMII             BIT(4)
196 #define      MVNETA_GMAC2_PORT_RESET             BIT(6)
197 #define MVNETA_GMAC_STATUS                       0x2c10
198 #define      MVNETA_GMAC_LINK_UP                 BIT(0)
199 #define      MVNETA_GMAC_SPEED_1000              BIT(1)
200 #define      MVNETA_GMAC_SPEED_100               BIT(2)
201 #define      MVNETA_GMAC_FULL_DUPLEX             BIT(3)
202 #define      MVNETA_GMAC_RX_FLOW_CTRL_ENABLE     BIT(4)
203 #define      MVNETA_GMAC_TX_FLOW_CTRL_ENABLE     BIT(5)
204 #define      MVNETA_GMAC_RX_FLOW_CTRL_ACTIVE     BIT(6)
205 #define      MVNETA_GMAC_TX_FLOW_CTRL_ACTIVE     BIT(7)
206 #define MVNETA_GMAC_AUTONEG_CONFIG               0x2c0c
207 #define      MVNETA_GMAC_FORCE_LINK_DOWN         BIT(0)
208 #define      MVNETA_GMAC_FORCE_LINK_PASS         BIT(1)
209 #define      MVNETA_GMAC_INBAND_AN_ENABLE        BIT(2)
210 #define      MVNETA_GMAC_CONFIG_MII_SPEED        BIT(5)
211 #define      MVNETA_GMAC_CONFIG_GMII_SPEED       BIT(6)
212 #define      MVNETA_GMAC_AN_SPEED_EN             BIT(7)
213 #define      MVNETA_GMAC_AN_FLOW_CTRL_EN         BIT(11)
214 #define      MVNETA_GMAC_CONFIG_FULL_DUPLEX      BIT(12)
215 #define      MVNETA_GMAC_AN_DUPLEX_EN            BIT(13)
216 #define MVNETA_MIB_COUNTERS_BASE                 0x3000
217 #define      MVNETA_MIB_LATE_COLLISION           0x7c
218 #define MVNETA_DA_FILT_SPEC_MCAST                0x3400
219 #define MVNETA_DA_FILT_OTH_MCAST                 0x3500
220 #define MVNETA_DA_FILT_UCAST_BASE                0x3600
221 #define MVNETA_TXQ_BASE_ADDR_REG(q)              (0x3c00 + ((q) << 2))
222 #define MVNETA_TXQ_SIZE_REG(q)                   (0x3c20 + ((q) << 2))
223 #define      MVNETA_TXQ_SENT_THRESH_ALL_MASK     0x3fff0000
224 #define      MVNETA_TXQ_SENT_THRESH_MASK(coal)   ((coal) << 16)
225 #define MVNETA_TXQ_UPDATE_REG(q)                 (0x3c60 + ((q) << 2))
226 #define      MVNETA_TXQ_DEC_SENT_SHIFT           16
227 #define MVNETA_TXQ_STATUS_REG(q)                 (0x3c40 + ((q) << 2))
228 #define      MVNETA_TXQ_SENT_DESC_SHIFT          16
229 #define      MVNETA_TXQ_SENT_DESC_MASK           0x3fff0000
230 #define MVNETA_PORT_TX_RESET                     0x3cf0
231 #define      MVNETA_PORT_TX_DMA_RESET            BIT(0)
232 #define MVNETA_TX_MTU                            0x3e0c
233 #define MVNETA_TX_TOKEN_SIZE                     0x3e14
234 #define      MVNETA_TX_TOKEN_SIZE_MAX            0xffffffff
235 #define MVNETA_TXQ_TOKEN_SIZE_REG(q)             (0x3e40 + ((q) << 2))
236 #define      MVNETA_TXQ_TOKEN_SIZE_MAX           0x7fffffff
237 
238 #define MVNETA_CAUSE_TXQ_SENT_DESC_ALL_MASK      0xff
239 
240 /* Descriptor ring Macros */
241 #define MVNETA_QUEUE_NEXT_DESC(q, index)        \
242         (((index) < (q)->last_desc) ? ((index) + 1) : 0)
243 
244 /* Various constants */
245 
246 /* Coalescing */
247 #define MVNETA_TXDONE_COAL_PKTS         0       /* interrupt per packet */
248 #define MVNETA_RX_COAL_PKTS             32
249 #define MVNETA_RX_COAL_USEC             100
250 
251 /* The two bytes Marvell header. Either contains a special value used
252  * by Marvell switches when a specific hardware mode is enabled (not
253  * supported by this driver) or is filled automatically by zeroes on
254  * the RX side. Those two bytes being at the front of the Ethernet
255  * header, they allow to have the IP header aligned on a 4 bytes
256  * boundary automatically: the hardware skips those two bytes on its
257  * own.
258  */
259 #define MVNETA_MH_SIZE                  2
260 
261 #define MVNETA_VLAN_TAG_LEN             4
262 
263 #define MVNETA_TX_CSUM_DEF_SIZE         1600
264 #define MVNETA_TX_CSUM_MAX_SIZE         9800
265 #define MVNETA_ACC_MODE_EXT1            1
266 #define MVNETA_ACC_MODE_EXT2            2
267 
268 #define MVNETA_MAX_DECODE_WIN           6
269 
270 /* Timeout constants */
271 #define MVNETA_TX_DISABLE_TIMEOUT_MSEC  1000
272 #define MVNETA_RX_DISABLE_TIMEOUT_MSEC  1000
273 #define MVNETA_TX_FIFO_EMPTY_TIMEOUT    10000
274 
275 #define MVNETA_TX_MTU_MAX               0x3ffff
276 
277 /* The RSS lookup table actually has 256 entries but we do not use
278  * them yet
279  */
280 #define MVNETA_RSS_LU_TABLE_SIZE        1
281 
282 /* TSO header size */
283 #define TSO_HEADER_SIZE 128
284 
285 /* Max number of Rx descriptors */
286 #define MVNETA_MAX_RXD 128
287 
288 /* Max number of Tx descriptors */
289 #define MVNETA_MAX_TXD 532
290 
291 /* Max number of allowed TCP segments for software TSO */
292 #define MVNETA_MAX_TSO_SEGS 100
293 
294 #define MVNETA_MAX_SKB_DESCS (MVNETA_MAX_TSO_SEGS * 2 + MAX_SKB_FRAGS)
295 
296 /* descriptor aligned size */
297 #define MVNETA_DESC_ALIGNED_SIZE        32
298 
299 /* Number of bytes to be taken into account by HW when putting incoming data
300  * to the buffers. It is needed in case NET_SKB_PAD exceeds maximum packet
301  * offset supported in MVNETA_RXQ_CONFIG_REG(q) registers.
302  */
303 #define MVNETA_RX_PKT_OFFSET_CORRECTION         64
304 
305 #define MVNETA_RX_PKT_SIZE(mtu) \
306         ALIGN((mtu) + MVNETA_MH_SIZE + MVNETA_VLAN_TAG_LEN + \
307               ETH_HLEN + ETH_FCS_LEN,                        \
308               cache_line_size())
309 
310 #define IS_TSO_HEADER(txq, addr) \
311         ((addr >= txq->tso_hdrs_phys) && \
312          (addr < txq->tso_hdrs_phys + txq->size * TSO_HEADER_SIZE))
313 
314 #define MVNETA_RX_GET_BM_POOL_ID(rxd) \
315         (((rxd)->status & MVNETA_RXD_BM_POOL_MASK) >> MVNETA_RXD_BM_POOL_SHIFT)
316 
317 struct mvneta_statistic {
318         unsigned short offset;
319         unsigned short type;
320         const char name[ETH_GSTRING_LEN];
321 };
322 
323 #define T_REG_32        32
324 #define T_REG_64        64
325 
326 static const struct mvneta_statistic mvneta_statistics[] = {
327         { 0x3000, T_REG_64, "good_octets_received", },
328         { 0x3010, T_REG_32, "good_frames_received", },
329         { 0x3008, T_REG_32, "bad_octets_received", },
330         { 0x3014, T_REG_32, "bad_frames_received", },
331         { 0x3018, T_REG_32, "broadcast_frames_received", },
332         { 0x301c, T_REG_32, "multicast_frames_received", },
333         { 0x3050, T_REG_32, "unrec_mac_control_received", },
334         { 0x3058, T_REG_32, "good_fc_received", },
335         { 0x305c, T_REG_32, "bad_fc_received", },
336         { 0x3060, T_REG_32, "undersize_received", },
337         { 0x3064, T_REG_32, "fragments_received", },
338         { 0x3068, T_REG_32, "oversize_received", },
339         { 0x306c, T_REG_32, "jabber_received", },
340         { 0x3070, T_REG_32, "mac_receive_error", },
341         { 0x3074, T_REG_32, "bad_crc_event", },
342         { 0x3078, T_REG_32, "collision", },
343         { 0x307c, T_REG_32, "late_collision", },
344         { 0x2484, T_REG_32, "rx_discard", },
345         { 0x2488, T_REG_32, "rx_overrun", },
346         { 0x3020, T_REG_32, "frames_64_octets", },
347         { 0x3024, T_REG_32, "frames_65_to_127_octets", },
348         { 0x3028, T_REG_32, "frames_128_to_255_octets", },
349         { 0x302c, T_REG_32, "frames_256_to_511_octets", },
350         { 0x3030, T_REG_32, "frames_512_to_1023_octets", },
351         { 0x3034, T_REG_32, "frames_1024_to_max_octets", },
352         { 0x3038, T_REG_64, "good_octets_sent", },
353         { 0x3040, T_REG_32, "good_frames_sent", },
354         { 0x3044, T_REG_32, "excessive_collision", },
355         { 0x3048, T_REG_32, "multicast_frames_sent", },
356         { 0x304c, T_REG_32, "broadcast_frames_sent", },
357         { 0x3054, T_REG_32, "fc_sent", },
358         { 0x300c, T_REG_32, "internal_mac_transmit_err", },
359 };
360 
361 struct mvneta_pcpu_stats {
362         struct  u64_stats_sync syncp;
363         u64     rx_packets;
364         u64     rx_bytes;
365         u64     tx_packets;
366         u64     tx_bytes;
367 };
368 
369 struct mvneta_pcpu_port {
370         /* Pointer to the shared port */
371         struct mvneta_port      *pp;
372 
373         /* Pointer to the CPU-local NAPI struct */
374         struct napi_struct      napi;
375 
376         /* Cause of the previous interrupt */
377         u32                     cause_rx_tx;
378 };
379 
380 struct mvneta_port {
381         u8 id;
382         struct mvneta_pcpu_port __percpu        *ports;
383         struct mvneta_pcpu_stats __percpu       *stats;
384 
385         int pkt_size;
386         unsigned int frag_size;
387         void __iomem *base;
388         struct mvneta_rx_queue *rxqs;
389         struct mvneta_tx_queue *txqs;
390         struct net_device *dev;
391         struct hlist_node node_online;
392         struct hlist_node node_dead;
393         int rxq_def;
394         /* Protect the access to the percpu interrupt registers,
395          * ensuring that the configuration remains coherent.
396          */
397         spinlock_t lock;
398         bool is_stopped;
399 
400         u32 cause_rx_tx;
401         struct napi_struct napi;
402 
403         /* Core clock */
404         struct clk *clk;
405         /* AXI clock */
406         struct clk *clk_bus;
407         u8 mcast_count[256];
408         u16 tx_ring_size;
409         u16 rx_ring_size;
410 
411         struct mii_bus *mii_bus;
412         phy_interface_t phy_interface;
413         struct device_node *phy_node;
414         unsigned int link;
415         unsigned int duplex;
416         unsigned int speed;
417         unsigned int tx_csum_limit;
418         unsigned int use_inband_status:1;
419 
420         struct mvneta_bm *bm_priv;
421         struct mvneta_bm_pool *pool_long;
422         struct mvneta_bm_pool *pool_short;
423         int bm_win_id;
424 
425         u64 ethtool_stats[ARRAY_SIZE(mvneta_statistics)];
426 
427         u32 indir[MVNETA_RSS_LU_TABLE_SIZE];
428 
429         /* Flags for special SoC configurations */
430         bool neta_armada3700;
431         u16 rx_offset_correction;
432 };
433 
434 /* The mvneta_tx_desc and mvneta_rx_desc structures describe the
435  * layout of the transmit and reception DMA descriptors, and their
436  * layout is therefore defined by the hardware design
437  */
438 
439 #define MVNETA_TX_L3_OFF_SHIFT  0
440 #define MVNETA_TX_IP_HLEN_SHIFT 8
441 #define MVNETA_TX_L4_UDP        BIT(16)
442 #define MVNETA_TX_L3_IP6        BIT(17)
443 #define MVNETA_TXD_IP_CSUM      BIT(18)
444 #define MVNETA_TXD_Z_PAD        BIT(19)
445 #define MVNETA_TXD_L_DESC       BIT(20)
446 #define MVNETA_TXD_F_DESC       BIT(21)
447 #define MVNETA_TXD_FLZ_DESC     (MVNETA_TXD_Z_PAD  | \
448                                  MVNETA_TXD_L_DESC | \
449                                  MVNETA_TXD_F_DESC)
450 #define MVNETA_TX_L4_CSUM_FULL  BIT(30)
451 #define MVNETA_TX_L4_CSUM_NOT   BIT(31)
452 
453 #define MVNETA_RXD_ERR_CRC              0x0
454 #define MVNETA_RXD_BM_POOL_SHIFT        13
455 #define MVNETA_RXD_BM_POOL_MASK         (BIT(13) | BIT(14))
456 #define MVNETA_RXD_ERR_SUMMARY          BIT(16)
457 #define MVNETA_RXD_ERR_OVERRUN          BIT(17)
458 #define MVNETA_RXD_ERR_LEN              BIT(18)
459 #define MVNETA_RXD_ERR_RESOURCE         (BIT(17) | BIT(18))
460 #define MVNETA_RXD_ERR_CODE_MASK        (BIT(17) | BIT(18))
461 #define MVNETA_RXD_L3_IP4               BIT(25)
462 #define MVNETA_RXD_FIRST_LAST_DESC      (BIT(26) | BIT(27))
463 #define MVNETA_RXD_L4_CSUM_OK           BIT(30)
464 
465 #if defined(__LITTLE_ENDIAN)
466 struct mvneta_tx_desc {
467         u32  command;           /* Options used by HW for packet transmitting.*/
468         u16  reserverd1;        /* csum_l4 (for future use)             */
469         u16  data_size;         /* Data size of transmitted packet in bytes */
470         u32  buf_phys_addr;     /* Physical addr of transmitted buffer  */
471         u32  reserved2;         /* hw_cmd - (for future use, PMT)       */
472         u32  reserved3[4];      /* Reserved - (for future use)          */
473 };
474 
475 struct mvneta_rx_desc {
476         u32  status;            /* Info about received packet           */
477         u16  reserved1;         /* pnc_info - (for future use, PnC)     */
478         u16  data_size;         /* Size of received packet in bytes     */
479 
480         u32  buf_phys_addr;     /* Physical address of the buffer       */
481         u32  reserved2;         /* pnc_flow_id  (for future use, PnC)   */
482 
483         u32  buf_cookie;        /* cookie for access to RX buffer in rx path */
484         u16  reserved3;         /* prefetch_cmd, for future use         */
485         u16  reserved4;         /* csum_l4 - (for future use, PnC)      */
486 
487         u32  reserved5;         /* pnc_extra PnC (for future use, PnC)  */
488         u32  reserved6;         /* hw_cmd (for future use, PnC and HWF) */
489 };
490 #else
491 struct mvneta_tx_desc {
492         u16  data_size;         /* Data size of transmitted packet in bytes */
493         u16  reserverd1;        /* csum_l4 (for future use)             */
494         u32  command;           /* Options used by HW for packet transmitting.*/
495         u32  reserved2;         /* hw_cmd - (for future use, PMT)       */
496         u32  buf_phys_addr;     /* Physical addr of transmitted buffer  */
497         u32  reserved3[4];      /* Reserved - (for future use)          */
498 };
499 
500 struct mvneta_rx_desc {
501         u16  data_size;         /* Size of received packet in bytes     */
502         u16  reserved1;         /* pnc_info - (for future use, PnC)     */
503         u32  status;            /* Info about received packet           */
504 
505         u32  reserved2;         /* pnc_flow_id  (for future use, PnC)   */
506         u32  buf_phys_addr;     /* Physical address of the buffer       */
507 
508         u16  reserved4;         /* csum_l4 - (for future use, PnC)      */
509         u16  reserved3;         /* prefetch_cmd, for future use         */
510         u32  buf_cookie;        /* cookie for access to RX buffer in rx path */
511 
512         u32  reserved5;         /* pnc_extra PnC (for future use, PnC)  */
513         u32  reserved6;         /* hw_cmd (for future use, PnC and HWF) */
514 };
515 #endif
516 
517 struct mvneta_tx_queue {
518         /* Number of this TX queue, in the range 0-7 */
519         u8 id;
520 
521         /* Number of TX DMA descriptors in the descriptor ring */
522         int size;
523 
524         /* Number of currently used TX DMA descriptor in the
525          * descriptor ring
526          */
527         int count;
528         int tx_stop_threshold;
529         int tx_wake_threshold;
530 
531         /* Array of transmitted skb */
532         struct sk_buff **tx_skb;
533 
534         /* Index of last TX DMA descriptor that was inserted */
535         int txq_put_index;
536 
537         /* Index of the TX DMA descriptor to be cleaned up */
538         int txq_get_index;
539 
540         u32 done_pkts_coal;
541 
542         /* Virtual address of the TX DMA descriptors array */
543         struct mvneta_tx_desc *descs;
544 
545         /* DMA address of the TX DMA descriptors array */
546         dma_addr_t descs_phys;
547 
548         /* Index of the last TX DMA descriptor */
549         int last_desc;
550 
551         /* Index of the next TX DMA descriptor to process */
552         int next_desc_to_proc;
553 
554         /* DMA buffers for TSO headers */
555         char *tso_hdrs;
556 
557         /* DMA address of TSO headers */
558         dma_addr_t tso_hdrs_phys;
559 
560         /* Affinity mask for CPUs*/
561         cpumask_t affinity_mask;
562 };
563 
564 struct mvneta_rx_queue {
565         /* rx queue number, in the range 0-7 */
566         u8 id;
567 
568         /* num of rx descriptors in the rx descriptor ring */
569         int size;
570 
571         /* counter of times when mvneta_refill() failed */
572         int missed;
573 
574         u32 pkts_coal;
575         u32 time_coal;
576 
577         /* Virtual address of the RX buffer */
578         void  **buf_virt_addr;
579 
580         /* Virtual address of the RX DMA descriptors array */
581         struct mvneta_rx_desc *descs;
582 
583         /* DMA address of the RX DMA descriptors array */
584         dma_addr_t descs_phys;
585 
586         /* Index of the last RX DMA descriptor */
587         int last_desc;
588 
589         /* Index of the next RX DMA descriptor to process */
590         int next_desc_to_proc;
591 };
592 
593 static enum cpuhp_state online_hpstate;
594 /* The hardware supports eight (8) rx queues, but we are only allowing
595  * the first one to be used. Therefore, let's just allocate one queue.
596  */
597 static int rxq_number = 8;
598 static int txq_number = 8;
599 
600 static int rxq_def;
601 
602 static int rx_copybreak __read_mostly = 256;
603 
604 /* HW BM need that each port be identify by a unique ID */
605 static int global_port_id;
606 
607 #define MVNETA_DRIVER_NAME "mvneta"
608 #define MVNETA_DRIVER_VERSION "1.0"
609 
610 /* Utility/helper methods */
611 
612 /* Write helper method */
613 static void mvreg_write(struct mvneta_port *pp, u32 offset, u32 data)
614 {
615         writel(data, pp->base + offset);
616 }
617 
618 /* Read helper method */
619 static u32 mvreg_read(struct mvneta_port *pp, u32 offset)
620 {
621         return readl(pp->base + offset);
622 }
623 
624 /* Increment txq get counter */
625 static void mvneta_txq_inc_get(struct mvneta_tx_queue *txq)
626 {
627         txq->txq_get_index++;
628         if (txq->txq_get_index == txq->size)
629                 txq->txq_get_index = 0;
630 }
631 
632 /* Increment txq put counter */
633 static void mvneta_txq_inc_put(struct mvneta_tx_queue *txq)
634 {
635         txq->txq_put_index++;
636         if (txq->txq_put_index == txq->size)
637                 txq->txq_put_index = 0;
638 }
639 
640 
641 /* Clear all MIB counters */
642 static void mvneta_mib_counters_clear(struct mvneta_port *pp)
643 {
644         int i;
645         u32 dummy;
646 
647         /* Perform dummy reads from MIB counters */
648         for (i = 0; i < MVNETA_MIB_LATE_COLLISION; i += 4)
649                 dummy = mvreg_read(pp, (MVNETA_MIB_COUNTERS_BASE + i));
650         dummy = mvreg_read(pp, MVNETA_RX_DISCARD_FRAME_COUNT);
651         dummy = mvreg_read(pp, MVNETA_OVERRUN_FRAME_COUNT);
652 }
653 
654 /* Get System Network Statistics */
655 static struct rtnl_link_stats64 *
656 mvneta_get_stats64(struct net_device *dev,
657                    struct rtnl_link_stats64 *stats)
658 {
659         struct mvneta_port *pp = netdev_priv(dev);
660         unsigned int start;
661         int cpu;
662 
663         for_each_possible_cpu(cpu) {
664                 struct mvneta_pcpu_stats *cpu_stats;
665                 u64 rx_packets;
666                 u64 rx_bytes;
667                 u64 tx_packets;
668                 u64 tx_bytes;
669 
670                 cpu_stats = per_cpu_ptr(pp->stats, cpu);
671                 do {
672                         start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
673                         rx_packets = cpu_stats->rx_packets;
674                         rx_bytes   = cpu_stats->rx_bytes;
675                         tx_packets = cpu_stats->tx_packets;
676                         tx_bytes   = cpu_stats->tx_bytes;
677                 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
678 
679                 stats->rx_packets += rx_packets;
680                 stats->rx_bytes   += rx_bytes;
681                 stats->tx_packets += tx_packets;
682                 stats->tx_bytes   += tx_bytes;
683         }
684 
685         stats->rx_errors        = dev->stats.rx_errors;
686         stats->rx_dropped       = dev->stats.rx_dropped;
687 
688         stats->tx_dropped       = dev->stats.tx_dropped;
689 
690         return stats;
691 }
692 
693 /* Rx descriptors helper methods */
694 
695 /* Checks whether the RX descriptor having this status is both the first
696  * and the last descriptor for the RX packet. Each RX packet is currently
697  * received through a single RX descriptor, so not having each RX
698  * descriptor with its first and last bits set is an error
699  */
700 static int mvneta_rxq_desc_is_first_last(u32 status)
701 {
702         return (status & MVNETA_RXD_FIRST_LAST_DESC) ==
703                 MVNETA_RXD_FIRST_LAST_DESC;
704 }
705 
706 /* Add number of descriptors ready to receive new packets */
707 static void mvneta_rxq_non_occup_desc_add(struct mvneta_port *pp,
708                                           struct mvneta_rx_queue *rxq,
709                                           int ndescs)
710 {
711         /* Only MVNETA_RXQ_ADD_NON_OCCUPIED_MAX (255) descriptors can
712          * be added at once
713          */
714         while (ndescs > MVNETA_RXQ_ADD_NON_OCCUPIED_MAX) {
715                 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id),
716                             (MVNETA_RXQ_ADD_NON_OCCUPIED_MAX <<
717                              MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT));
718                 ndescs -= MVNETA_RXQ_ADD_NON_OCCUPIED_MAX;
719         }
720 
721         mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id),
722                     (ndescs << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT));
723 }
724 
725 /* Get number of RX descriptors occupied by received packets */
726 static int mvneta_rxq_busy_desc_num_get(struct mvneta_port *pp,
727                                         struct mvneta_rx_queue *rxq)
728 {
729         u32 val;
730 
731         val = mvreg_read(pp, MVNETA_RXQ_STATUS_REG(rxq->id));
732         return val & MVNETA_RXQ_OCCUPIED_ALL_MASK;
733 }
734 
735 /* Update num of rx desc called upon return from rx path or
736  * from mvneta_rxq_drop_pkts().
737  */
738 static void mvneta_rxq_desc_num_update(struct mvneta_port *pp,
739                                        struct mvneta_rx_queue *rxq,
740                                        int rx_done, int rx_filled)
741 {
742         u32 val;
743 
744         if ((rx_done <= 0xff) && (rx_filled <= 0xff)) {
745                 val = rx_done |
746                   (rx_filled << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT);
747                 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), val);
748                 return;
749         }
750 
751         /* Only 255 descriptors can be added at once */
752         while ((rx_done > 0) || (rx_filled > 0)) {
753                 if (rx_done <= 0xff) {
754                         val = rx_done;
755                         rx_done = 0;
756                 } else {
757                         val = 0xff;
758                         rx_done -= 0xff;
759                 }
760                 if (rx_filled <= 0xff) {
761                         val |= rx_filled << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT;
762                         rx_filled = 0;
763                 } else {
764                         val |= 0xff << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT;
765                         rx_filled -= 0xff;
766                 }
767                 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), val);
768         }
769 }
770 
771 /* Get pointer to next RX descriptor to be processed by SW */
772 static struct mvneta_rx_desc *
773 mvneta_rxq_next_desc_get(struct mvneta_rx_queue *rxq)
774 {
775         int rx_desc = rxq->next_desc_to_proc;
776 
777         rxq->next_desc_to_proc = MVNETA_QUEUE_NEXT_DESC(rxq, rx_desc);
778         prefetch(rxq->descs + rxq->next_desc_to_proc);
779         return rxq->descs + rx_desc;
780 }
781 
782 /* Change maximum receive size of the port. */
783 static void mvneta_max_rx_size_set(struct mvneta_port *pp, int max_rx_size)
784 {
785         u32 val;
786 
787         val =  mvreg_read(pp, MVNETA_GMAC_CTRL_0);
788         val &= ~MVNETA_GMAC_MAX_RX_SIZE_MASK;
789         val |= ((max_rx_size - MVNETA_MH_SIZE) / 2) <<
790                 MVNETA_GMAC_MAX_RX_SIZE_SHIFT;
791         mvreg_write(pp, MVNETA_GMAC_CTRL_0, val);
792 }
793 
794 
795 /* Set rx queue offset */
796 static void mvneta_rxq_offset_set(struct mvneta_port *pp,
797                                   struct mvneta_rx_queue *rxq,
798                                   int offset)
799 {
800         u32 val;
801 
802         val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id));
803         val &= ~MVNETA_RXQ_PKT_OFFSET_ALL_MASK;
804 
805         /* Offset is in */
806         val |= MVNETA_RXQ_PKT_OFFSET_MASK(offset >> 3);
807         mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val);
808 }
809 
810 
811 /* Tx descriptors helper methods */
812 
813 /* Update HW with number of TX descriptors to be sent */
814 static void mvneta_txq_pend_desc_add(struct mvneta_port *pp,
815                                      struct mvneta_tx_queue *txq,
816                                      int pend_desc)
817 {
818         u32 val;
819 
820         /* Only 255 descriptors can be added at once ; Assume caller
821          * process TX desriptors in quanta less than 256
822          */
823         val = pend_desc;
824         mvreg_write(pp, MVNETA_TXQ_UPDATE_REG(txq->id), val);
825 }
826 
827 /* Get pointer to next TX descriptor to be processed (send) by HW */
828 static struct mvneta_tx_desc *
829 mvneta_txq_next_desc_get(struct mvneta_tx_queue *txq)
830 {
831         int tx_desc = txq->next_desc_to_proc;
832 
833         txq->next_desc_to_proc = MVNETA_QUEUE_NEXT_DESC(txq, tx_desc);
834         return txq->descs + tx_desc;
835 }
836 
837 /* Release the last allocated TX descriptor. Useful to handle DMA
838  * mapping failures in the TX path.
839  */
840 static void mvneta_txq_desc_put(struct mvneta_tx_queue *txq)
841 {
842         if (txq->next_desc_to_proc == 0)
843                 txq->next_desc_to_proc = txq->last_desc - 1;
844         else
845                 txq->next_desc_to_proc--;
846 }
847 
848 /* Set rxq buf size */
849 static void mvneta_rxq_buf_size_set(struct mvneta_port *pp,
850                                     struct mvneta_rx_queue *rxq,
851                                     int buf_size)
852 {
853         u32 val;
854 
855         val = mvreg_read(pp, MVNETA_RXQ_SIZE_REG(rxq->id));
856 
857         val &= ~MVNETA_RXQ_BUF_SIZE_MASK;
858         val |= ((buf_size >> 3) << MVNETA_RXQ_BUF_SIZE_SHIFT);
859 
860         mvreg_write(pp, MVNETA_RXQ_SIZE_REG(rxq->id), val);
861 }
862 
863 /* Disable buffer management (BM) */
864 static void mvneta_rxq_bm_disable(struct mvneta_port *pp,
865                                   struct mvneta_rx_queue *rxq)
866 {
867         u32 val;
868 
869         val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id));
870         val &= ~MVNETA_RXQ_HW_BUF_ALLOC;
871         mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val);
872 }
873 
874 /* Enable buffer management (BM) */
875 static void mvneta_rxq_bm_enable(struct mvneta_port *pp,
876                                  struct mvneta_rx_queue *rxq)
877 {
878         u32 val;
879 
880         val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id));
881         val |= MVNETA_RXQ_HW_BUF_ALLOC;
882         mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val);
883 }
884 
885 /* Notify HW about port's assignment of pool for bigger packets */
886 static void mvneta_rxq_long_pool_set(struct mvneta_port *pp,
887                                      struct mvneta_rx_queue *rxq)
888 {
889         u32 val;
890 
891         val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id));
892         val &= ~MVNETA_RXQ_LONG_POOL_ID_MASK;
893         val |= (pp->pool_long->id << MVNETA_RXQ_LONG_POOL_ID_SHIFT);
894 
895         mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val);
896 }
897 
898 /* Notify HW about port's assignment of pool for smaller packets */
899 static void mvneta_rxq_short_pool_set(struct mvneta_port *pp,
900                                       struct mvneta_rx_queue *rxq)
901 {
902         u32 val;
903 
904         val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id));
905         val &= ~MVNETA_RXQ_SHORT_POOL_ID_MASK;
906         val |= (pp->pool_short->id << MVNETA_RXQ_SHORT_POOL_ID_SHIFT);
907 
908         mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val);
909 }
910 
911 /* Set port's receive buffer size for assigned BM pool */
912 static inline void mvneta_bm_pool_bufsize_set(struct mvneta_port *pp,
913                                               int buf_size,
914                                               u8 pool_id)
915 {
916         u32 val;
917 
918         if (!IS_ALIGNED(buf_size, 8)) {
919                 dev_warn(pp->dev->dev.parent,
920                          "illegal buf_size value %d, round to %d\n",
921                          buf_size, ALIGN(buf_size, 8));
922                 buf_size = ALIGN(buf_size, 8);
923         }
924 
925         val = mvreg_read(pp, MVNETA_PORT_POOL_BUFFER_SZ_REG(pool_id));
926         val |= buf_size & MVNETA_PORT_POOL_BUFFER_SZ_MASK;
927         mvreg_write(pp, MVNETA_PORT_POOL_BUFFER_SZ_REG(pool_id), val);
928 }
929 
930 /* Configure MBUS window in order to enable access BM internal SRAM */
931 static int mvneta_mbus_io_win_set(struct mvneta_port *pp, u32 base, u32 wsize,
932                                   u8 target, u8 attr)
933 {
934         u32 win_enable, win_protect;
935         int i;
936 
937         win_enable = mvreg_read(pp, MVNETA_BASE_ADDR_ENABLE);
938 
939         if (pp->bm_win_id < 0) {
940                 /* Find first not occupied window */
941                 for (i = 0; i < MVNETA_MAX_DECODE_WIN; i++) {
942                         if (win_enable & (1 << i)) {
943                                 pp->bm_win_id = i;
944                                 break;
945                         }
946                 }
947                 if (i == MVNETA_MAX_DECODE_WIN)
948                         return -ENOMEM;
949         } else {
950                 i = pp->bm_win_id;
951         }
952 
953         mvreg_write(pp, MVNETA_WIN_BASE(i), 0);
954         mvreg_write(pp, MVNETA_WIN_SIZE(i), 0);
955 
956         if (i < 4)
957                 mvreg_write(pp, MVNETA_WIN_REMAP(i), 0);
958 
959         mvreg_write(pp, MVNETA_WIN_BASE(i), (base & 0xffff0000) |
960                     (attr << 8) | target);
961 
962         mvreg_write(pp, MVNETA_WIN_SIZE(i), (wsize - 1) & 0xffff0000);
963 
964         win_protect = mvreg_read(pp, MVNETA_ACCESS_PROTECT_ENABLE);
965         win_protect |= 3 << (2 * i);
966         mvreg_write(pp, MVNETA_ACCESS_PROTECT_ENABLE, win_protect);
967 
968         win_enable &= ~(1 << i);
969         mvreg_write(pp, MVNETA_BASE_ADDR_ENABLE, win_enable);
970 
971         return 0;
972 }
973 
974 static  int mvneta_bm_port_mbus_init(struct mvneta_port *pp)
975 {
976         u32 wsize;
977         u8 target, attr;
978         int err;
979 
980         /* Get BM window information */
981         err = mvebu_mbus_get_io_win_info(pp->bm_priv->bppi_phys_addr, &wsize,
982                                          &target, &attr);
983         if (err < 0)
984                 return err;
985 
986         pp->bm_win_id = -1;
987 
988         /* Open NETA -> BM window */
989         err = mvneta_mbus_io_win_set(pp, pp->bm_priv->bppi_phys_addr, wsize,
990                                      target, attr);
991         if (err < 0) {
992                 netdev_info(pp->dev, "fail to configure mbus window to BM\n");
993                 return err;
994         }
995         return 0;
996 }
997 
998 /* Assign and initialize pools for port. In case of fail
999  * buffer manager will remain disabled for current port.
1000  */
1001 static int mvneta_bm_port_init(struct platform_device *pdev,
1002                                struct mvneta_port *pp)
1003 {
1004         struct device_node *dn = pdev->dev.of_node;
1005         u32 long_pool_id, short_pool_id;
1006 
1007         if (!pp->neta_armada3700) {
1008                 int ret;
1009 
1010                 ret = mvneta_bm_port_mbus_init(pp);
1011                 if (ret)
1012                         return ret;
1013         }
1014 
1015         if (of_property_read_u32(dn, "bm,pool-long", &long_pool_id)) {
1016                 netdev_info(pp->dev, "missing long pool id\n");
1017                 return -EINVAL;
1018         }
1019 
1020         /* Create port's long pool depending on mtu */
1021         pp->pool_long = mvneta_bm_pool_use(pp->bm_priv, long_pool_id,
1022                                            MVNETA_BM_LONG, pp->id,
1023                                            MVNETA_RX_PKT_SIZE(pp->dev->mtu));
1024         if (!pp->pool_long) {
1025                 netdev_info(pp->dev, "fail to obtain long pool for port\n");
1026                 return -ENOMEM;
1027         }
1028 
1029         pp->pool_long->port_map |= 1 << pp->id;
1030 
1031         mvneta_bm_pool_bufsize_set(pp, pp->pool_long->buf_size,
1032                                    pp->pool_long->id);
1033 
1034         /* If short pool id is not defined, assume using single pool */
1035         if (of_property_read_u32(dn, "bm,pool-short", &short_pool_id))
1036                 short_pool_id = long_pool_id;
1037 
1038         /* Create port's short pool */
1039         pp->pool_short = mvneta_bm_pool_use(pp->bm_priv, short_pool_id,
1040                                             MVNETA_BM_SHORT, pp->id,
1041                                             MVNETA_BM_SHORT_PKT_SIZE);
1042         if (!pp->pool_short) {
1043                 netdev_info(pp->dev, "fail to obtain short pool for port\n");
1044                 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id);
1045                 return -ENOMEM;
1046         }
1047 
1048         if (short_pool_id != long_pool_id) {
1049                 pp->pool_short->port_map |= 1 << pp->id;
1050                 mvneta_bm_pool_bufsize_set(pp, pp->pool_short->buf_size,
1051                                            pp->pool_short->id);
1052         }
1053 
1054         return 0;
1055 }
1056 
1057 /* Update settings of a pool for bigger packets */
1058 static void mvneta_bm_update_mtu(struct mvneta_port *pp, int mtu)
1059 {
1060         struct mvneta_bm_pool *bm_pool = pp->pool_long;
1061         struct hwbm_pool *hwbm_pool = &bm_pool->hwbm_pool;
1062         int num;
1063 
1064         /* Release all buffers from long pool */
1065         mvneta_bm_bufs_free(pp->bm_priv, bm_pool, 1 << pp->id);
1066         if (hwbm_pool->buf_num) {
1067                 WARN(1, "cannot free all buffers in pool %d\n",
1068                      bm_pool->id);
1069                 goto bm_mtu_err;
1070         }
1071 
1072         bm_pool->pkt_size = MVNETA_RX_PKT_SIZE(mtu);
1073         bm_pool->buf_size = MVNETA_RX_BUF_SIZE(bm_pool->pkt_size);
1074         hwbm_pool->frag_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
1075                         SKB_DATA_ALIGN(MVNETA_RX_BUF_SIZE(bm_pool->pkt_size));
1076 
1077         /* Fill entire long pool */
1078         num = hwbm_pool_add(hwbm_pool, hwbm_pool->size, GFP_ATOMIC);
1079         if (num != hwbm_pool->size) {
1080                 WARN(1, "pool %d: %d of %d allocated\n",
1081                      bm_pool->id, num, hwbm_pool->size);
1082                 goto bm_mtu_err;
1083         }
1084         mvneta_bm_pool_bufsize_set(pp, bm_pool->buf_size, bm_pool->id);
1085 
1086         return;
1087 
1088 bm_mtu_err:
1089         mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id);
1090         mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_short, 1 << pp->id);
1091 
1092         pp->bm_priv = NULL;
1093         mvreg_write(pp, MVNETA_ACC_MODE, MVNETA_ACC_MODE_EXT1);
1094         netdev_info(pp->dev, "fail to update MTU, fall back to software BM\n");
1095 }
1096 
1097 /* Start the Ethernet port RX and TX activity */
1098 static void mvneta_port_up(struct mvneta_port *pp)
1099 {
1100         int queue;
1101         u32 q_map;
1102 
1103         /* Enable all initialized TXs. */
1104         q_map = 0;
1105         for (queue = 0; queue < txq_number; queue++) {
1106                 struct mvneta_tx_queue *txq = &pp->txqs[queue];
1107                 if (txq->descs != NULL)
1108                         q_map |= (1 << queue);
1109         }
1110         mvreg_write(pp, MVNETA_TXQ_CMD, q_map);
1111 
1112         /* Enable all initialized RXQs. */
1113         for (queue = 0; queue < rxq_number; queue++) {
1114                 struct mvneta_rx_queue *rxq = &pp->rxqs[queue];
1115 
1116                 if (rxq->descs != NULL)
1117                         q_map |= (1 << queue);
1118         }
1119         mvreg_write(pp, MVNETA_RXQ_CMD, q_map);
1120 }
1121 
1122 /* Stop the Ethernet port activity */
1123 static void mvneta_port_down(struct mvneta_port *pp)
1124 {
1125         u32 val;
1126         int count;
1127 
1128         /* Stop Rx port activity. Check port Rx activity. */
1129         val = mvreg_read(pp, MVNETA_RXQ_CMD) & MVNETA_RXQ_ENABLE_MASK;
1130 
1131         /* Issue stop command for active channels only */
1132         if (val != 0)
1133                 mvreg_write(pp, MVNETA_RXQ_CMD,
1134                             val << MVNETA_RXQ_DISABLE_SHIFT);
1135 
1136         /* Wait for all Rx activity to terminate. */
1137         count = 0;
1138         do {
1139                 if (count++ >= MVNETA_RX_DISABLE_TIMEOUT_MSEC) {
1140                         netdev_warn(pp->dev,
1141                                     "TIMEOUT for RX stopped ! rx_queue_cmd: 0x%08x\n",
1142                                     val);
1143                         break;
1144                 }
1145                 mdelay(1);
1146 
1147                 val = mvreg_read(pp, MVNETA_RXQ_CMD);
1148         } while (val & MVNETA_RXQ_ENABLE_MASK);
1149 
1150         /* Stop Tx port activity. Check port Tx activity. Issue stop
1151          * command for active channels only
1152          */
1153         val = (mvreg_read(pp, MVNETA_TXQ_CMD)) & MVNETA_TXQ_ENABLE_MASK;
1154 
1155         if (val != 0)
1156                 mvreg_write(pp, MVNETA_TXQ_CMD,
1157                             (val << MVNETA_TXQ_DISABLE_SHIFT));
1158 
1159         /* Wait for all Tx activity to terminate. */
1160         count = 0;
1161         do {
1162                 if (count++ >= MVNETA_TX_DISABLE_TIMEOUT_MSEC) {
1163                         netdev_warn(pp->dev,
1164                                     "TIMEOUT for TX stopped status=0x%08x\n",
1165                                     val);
1166                         break;
1167                 }
1168                 mdelay(1);
1169 
1170                 /* Check TX Command reg that all Txqs are stopped */
1171                 val = mvreg_read(pp, MVNETA_TXQ_CMD);
1172 
1173         } while (val & MVNETA_TXQ_ENABLE_MASK);
1174 
1175         /* Double check to verify that TX FIFO is empty */
1176         count = 0;
1177         do {
1178                 if (count++ >= MVNETA_TX_FIFO_EMPTY_TIMEOUT) {
1179                         netdev_warn(pp->dev,
1180                                     "TX FIFO empty timeout status=0x%08x\n",
1181                                     val);
1182                         break;
1183                 }
1184                 mdelay(1);
1185 
1186                 val = mvreg_read(pp, MVNETA_PORT_STATUS);
1187         } while (!(val & MVNETA_TX_FIFO_EMPTY) &&
1188                  (val & MVNETA_TX_IN_PRGRS));
1189 
1190         udelay(200);
1191 }
1192 
1193 /* Enable the port by setting the port enable bit of the MAC control register */
1194 static void mvneta_port_enable(struct mvneta_port *pp)
1195 {
1196         u32 val;
1197 
1198         /* Enable port */
1199         val = mvreg_read(pp, MVNETA_GMAC_CTRL_0);
1200         val |= MVNETA_GMAC0_PORT_ENABLE;
1201         mvreg_write(pp, MVNETA_GMAC_CTRL_0, val);
1202 }
1203 
1204 /* Disable the port and wait for about 200 usec before retuning */
1205 static void mvneta_port_disable(struct mvneta_port *pp)
1206 {
1207         u32 val;
1208 
1209         /* Reset the Enable bit in the Serial Control Register */
1210         val = mvreg_read(pp, MVNETA_GMAC_CTRL_0);
1211         val &= ~MVNETA_GMAC0_PORT_ENABLE;
1212         mvreg_write(pp, MVNETA_GMAC_CTRL_0, val);
1213 
1214         udelay(200);
1215 }
1216 
1217 /* Multicast tables methods */
1218 
1219 /* Set all entries in Unicast MAC Table; queue==-1 means reject all */
1220 static void mvneta_set_ucast_table(struct mvneta_port *pp, int queue)
1221 {
1222         int offset;
1223         u32 val;
1224 
1225         if (queue == -1) {
1226                 val = 0;
1227         } else {
1228                 val = 0x1 | (queue << 1);
1229                 val |= (val << 24) | (val << 16) | (val << 8);
1230         }
1231 
1232         for (offset = 0; offset <= 0xc; offset += 4)
1233                 mvreg_write(pp, MVNETA_DA_FILT_UCAST_BASE + offset, val);
1234 }
1235 
1236 /* Set all entries in Special Multicast MAC Table; queue==-1 means reject all */
1237 static void mvneta_set_special_mcast_table(struct mvneta_port *pp, int queue)
1238 {
1239         int offset;
1240         u32 val;
1241 
1242         if (queue == -1) {
1243                 val = 0;
1244         } else {
1245                 val = 0x1 | (queue << 1);
1246                 val |= (val << 24) | (val << 16) | (val << 8);
1247         }
1248 
1249         for (offset = 0; offset <= 0xfc; offset += 4)
1250                 mvreg_write(pp, MVNETA_DA_FILT_SPEC_MCAST + offset, val);
1251 
1252 }
1253 
1254 /* Set all entries in Other Multicast MAC Table. queue==-1 means reject all */
1255 static void mvneta_set_other_mcast_table(struct mvneta_port *pp, int queue)
1256 {
1257         int offset;
1258         u32 val;
1259 
1260         if (queue == -1) {
1261                 memset(pp->mcast_count, 0, sizeof(pp->mcast_count));
1262                 val = 0;
1263         } else {
1264                 memset(pp->mcast_count, 1, sizeof(pp->mcast_count));
1265                 val = 0x1 | (queue << 1);
1266                 val |= (val << 24) | (val << 16) | (val << 8);
1267         }
1268 
1269         for (offset = 0; offset <= 0xfc; offset += 4)
1270                 mvreg_write(pp, MVNETA_DA_FILT_OTH_MCAST + offset, val);
1271 }
1272 
1273 static void mvneta_set_autoneg(struct mvneta_port *pp, int enable)
1274 {
1275         u32 val;
1276 
1277         if (enable) {
1278                 val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
1279                 val &= ~(MVNETA_GMAC_FORCE_LINK_PASS |
1280                          MVNETA_GMAC_FORCE_LINK_DOWN |
1281                          MVNETA_GMAC_AN_FLOW_CTRL_EN);
1282                 val |= MVNETA_GMAC_INBAND_AN_ENABLE |
1283                        MVNETA_GMAC_AN_SPEED_EN |
1284                        MVNETA_GMAC_AN_DUPLEX_EN;
1285                 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
1286 
1287                 val = mvreg_read(pp, MVNETA_GMAC_CLOCK_DIVIDER);
1288                 val |= MVNETA_GMAC_1MS_CLOCK_ENABLE;
1289                 mvreg_write(pp, MVNETA_GMAC_CLOCK_DIVIDER, val);
1290 
1291                 val = mvreg_read(pp, MVNETA_GMAC_CTRL_2);
1292                 val |= MVNETA_GMAC2_INBAND_AN_ENABLE;
1293                 mvreg_write(pp, MVNETA_GMAC_CTRL_2, val);
1294         } else {
1295                 val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
1296                 val &= ~(MVNETA_GMAC_INBAND_AN_ENABLE |
1297                        MVNETA_GMAC_AN_SPEED_EN |
1298                        MVNETA_GMAC_AN_DUPLEX_EN);
1299                 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
1300 
1301                 val = mvreg_read(pp, MVNETA_GMAC_CLOCK_DIVIDER);
1302                 val &= ~MVNETA_GMAC_1MS_CLOCK_ENABLE;
1303                 mvreg_write(pp, MVNETA_GMAC_CLOCK_DIVIDER, val);
1304 
1305                 val = mvreg_read(pp, MVNETA_GMAC_CTRL_2);
1306                 val &= ~MVNETA_GMAC2_INBAND_AN_ENABLE;
1307                 mvreg_write(pp, MVNETA_GMAC_CTRL_2, val);
1308         }
1309 }
1310 
1311 static void mvneta_percpu_unmask_interrupt(void *arg)
1312 {
1313         struct mvneta_port *pp = arg;
1314 
1315         /* All the queue are unmasked, but actually only the ones
1316          * mapped to this CPU will be unmasked
1317          */
1318         mvreg_write(pp, MVNETA_INTR_NEW_MASK,
1319                     MVNETA_RX_INTR_MASK_ALL |
1320                     MVNETA_TX_INTR_MASK_ALL |
1321                     MVNETA_MISCINTR_INTR_MASK);
1322 }
1323 
1324 static void mvneta_percpu_mask_interrupt(void *arg)
1325 {
1326         struct mvneta_port *pp = arg;
1327 
1328         /* All the queue are masked, but actually only the ones
1329          * mapped to this CPU will be masked
1330          */
1331         mvreg_write(pp, MVNETA_INTR_NEW_MASK, 0);
1332         mvreg_write(pp, MVNETA_INTR_OLD_MASK, 0);
1333         mvreg_write(pp, MVNETA_INTR_MISC_MASK, 0);
1334 }
1335 
1336 static void mvneta_percpu_clear_intr_cause(void *arg)
1337 {
1338         struct mvneta_port *pp = arg;
1339 
1340         /* All the queue are cleared, but actually only the ones
1341          * mapped to this CPU will be cleared
1342          */
1343         mvreg_write(pp, MVNETA_INTR_NEW_CAUSE, 0);
1344         mvreg_write(pp, MVNETA_INTR_MISC_CAUSE, 0);
1345         mvreg_write(pp, MVNETA_INTR_OLD_CAUSE, 0);
1346 }
1347 
1348 /* This method sets defaults to the NETA port:
1349  *      Clears interrupt Cause and Mask registers.
1350  *      Clears all MAC tables.
1351  *      Sets defaults to all registers.
1352  *      Resets RX and TX descriptor rings.
1353  *      Resets PHY.
1354  * This method can be called after mvneta_port_down() to return the port
1355  *      settings to defaults.
1356  */
1357 static void mvneta_defaults_set(struct mvneta_port *pp)
1358 {
1359         int cpu;
1360         int queue;
1361         u32 val;
1362         int max_cpu = num_present_cpus();
1363 
1364         /* Clear all Cause registers */
1365         on_each_cpu(mvneta_percpu_clear_intr_cause, pp, true);
1366 
1367         /* Mask all interrupts */
1368         on_each_cpu(mvneta_percpu_mask_interrupt, pp, true);
1369         mvreg_write(pp, MVNETA_INTR_ENABLE, 0);
1370 
1371         /* Enable MBUS Retry bit16 */
1372         mvreg_write(pp, MVNETA_MBUS_RETRY, 0x20);
1373 
1374         /* Set CPU queue access map. CPUs are assigned to the RX and
1375          * TX queues modulo their number. If there is only one TX
1376          * queue then it is assigned to the CPU associated to the
1377          * default RX queue.
1378          */
1379         for_each_present_cpu(cpu) {
1380                 int rxq_map = 0, txq_map = 0;
1381                 int rxq, txq;
1382                 if (!pp->neta_armada3700) {
1383                         for (rxq = 0; rxq < rxq_number; rxq++)
1384                                 if ((rxq % max_cpu) == cpu)
1385                                         rxq_map |= MVNETA_CPU_RXQ_ACCESS(rxq);
1386 
1387                         for (txq = 0; txq < txq_number; txq++)
1388                                 if ((txq % max_cpu) == cpu)
1389                                         txq_map |= MVNETA_CPU_TXQ_ACCESS(txq);
1390 
1391                         /* With only one TX queue we configure a special case
1392                          * which will allow to get all the irq on a single
1393                          * CPU
1394                          */
1395                         if (txq_number == 1)
1396                                 txq_map = (cpu == pp->rxq_def) ?
1397                                         MVNETA_CPU_TXQ_ACCESS(1) : 0;
1398 
1399                 } else {
1400                         txq_map = MVNETA_CPU_TXQ_ACCESS_ALL_MASK;
1401                         rxq_map = MVNETA_CPU_RXQ_ACCESS_ALL_MASK;
1402                 }
1403 
1404                 mvreg_write(pp, MVNETA_CPU_MAP(cpu), rxq_map | txq_map);
1405         }
1406 
1407         /* Reset RX and TX DMAs */
1408         mvreg_write(pp, MVNETA_PORT_RX_RESET, MVNETA_PORT_RX_DMA_RESET);
1409         mvreg_write(pp, MVNETA_PORT_TX_RESET, MVNETA_PORT_TX_DMA_RESET);
1410 
1411         /* Disable Legacy WRR, Disable EJP, Release from reset */
1412         mvreg_write(pp, MVNETA_TXQ_CMD_1, 0);
1413         for (queue = 0; queue < txq_number; queue++) {
1414                 mvreg_write(pp, MVETH_TXQ_TOKEN_COUNT_REG(queue), 0);
1415                 mvreg_write(pp, MVETH_TXQ_TOKEN_CFG_REG(queue), 0);
1416         }
1417 
1418         mvreg_write(pp, MVNETA_PORT_TX_RESET, 0);
1419         mvreg_write(pp, MVNETA_PORT_RX_RESET, 0);
1420 
1421         /* Set Port Acceleration Mode */
1422         if (pp->bm_priv)
1423                 /* HW buffer management + legacy parser */
1424                 val = MVNETA_ACC_MODE_EXT2;
1425         else
1426                 /* SW buffer management + legacy parser */
1427                 val = MVNETA_ACC_MODE_EXT1;
1428         mvreg_write(pp, MVNETA_ACC_MODE, val);
1429 
1430         if (pp->bm_priv)
1431                 mvreg_write(pp, MVNETA_BM_ADDRESS, pp->bm_priv->bppi_phys_addr);
1432 
1433         /* Update val of portCfg register accordingly with all RxQueue types */
1434         val = MVNETA_PORT_CONFIG_DEFL_VALUE(pp->rxq_def);
1435         mvreg_write(pp, MVNETA_PORT_CONFIG, val);
1436 
1437         val = 0;
1438         mvreg_write(pp, MVNETA_PORT_CONFIG_EXTEND, val);
1439         mvreg_write(pp, MVNETA_RX_MIN_FRAME_SIZE, 64);
1440 
1441         /* Build PORT_SDMA_CONFIG_REG */
1442         val = 0;
1443 
1444         /* Default burst size */
1445         val |= MVNETA_TX_BRST_SZ_MASK(MVNETA_SDMA_BRST_SIZE_16);
1446         val |= MVNETA_RX_BRST_SZ_MASK(MVNETA_SDMA_BRST_SIZE_16);
1447         val |= MVNETA_RX_NO_DATA_SWAP | MVNETA_TX_NO_DATA_SWAP;
1448 
1449 #if defined(__BIG_ENDIAN)
1450         val |= MVNETA_DESC_SWAP;
1451 #endif
1452 
1453         /* Assign port SDMA configuration */
1454         mvreg_write(pp, MVNETA_SDMA_CONFIG, val);
1455 
1456         /* Disable PHY polling in hardware, since we're using the
1457          * kernel phylib to do this.
1458          */
1459         val = mvreg_read(pp, MVNETA_UNIT_CONTROL);
1460         val &= ~MVNETA_PHY_POLLING_ENABLE;
1461         mvreg_write(pp, MVNETA_UNIT_CONTROL, val);
1462 
1463         mvneta_set_autoneg(pp, pp->use_inband_status);
1464         mvneta_set_ucast_table(pp, -1);
1465         mvneta_set_special_mcast_table(pp, -1);
1466         mvneta_set_other_mcast_table(pp, -1);
1467 
1468         /* Set port interrupt enable register - default enable all */
1469         mvreg_write(pp, MVNETA_INTR_ENABLE,
1470                     (MVNETA_RXQ_INTR_ENABLE_ALL_MASK
1471                      | MVNETA_TXQ_INTR_ENABLE_ALL_MASK));
1472 
1473         mvneta_mib_counters_clear(pp);
1474 }
1475 
1476 /* Set max sizes for tx queues */
1477 static void mvneta_txq_max_tx_size_set(struct mvneta_port *pp, int max_tx_size)
1478 
1479 {
1480         u32 val, size, mtu;
1481         int queue;
1482 
1483         mtu = max_tx_size * 8;
1484         if (mtu > MVNETA_TX_MTU_MAX)
1485                 mtu = MVNETA_TX_MTU_MAX;
1486 
1487         /* Set MTU */
1488         val = mvreg_read(pp, MVNETA_TX_MTU);
1489         val &= ~MVNETA_TX_MTU_MAX;
1490         val |= mtu;
1491         mvreg_write(pp, MVNETA_TX_MTU, val);
1492 
1493         /* TX token size and all TXQs token size must be larger that MTU */
1494         val = mvreg_read(pp, MVNETA_TX_TOKEN_SIZE);
1495 
1496         size = val & MVNETA_TX_TOKEN_SIZE_MAX;
1497         if (size < mtu) {
1498                 size = mtu;
1499                 val &= ~MVNETA_TX_TOKEN_SIZE_MAX;
1500                 val |= size;
1501                 mvreg_write(pp, MVNETA_TX_TOKEN_SIZE, val);
1502         }
1503         for (queue = 0; queue < txq_number; queue++) {
1504                 val = mvreg_read(pp, MVNETA_TXQ_TOKEN_SIZE_REG(queue));
1505 
1506                 size = val & MVNETA_TXQ_TOKEN_SIZE_MAX;
1507                 if (size < mtu) {
1508                         size = mtu;
1509                         val &= ~MVNETA_TXQ_TOKEN_SIZE_MAX;
1510                         val |= size;
1511                         mvreg_write(pp, MVNETA_TXQ_TOKEN_SIZE_REG(queue), val);
1512                 }
1513         }
1514 }
1515 
1516 /* Set unicast address */
1517 static void mvneta_set_ucast_addr(struct mvneta_port *pp, u8 last_nibble,
1518                                   int queue)
1519 {
1520         unsigned int unicast_reg;
1521         unsigned int tbl_offset;
1522         unsigned int reg_offset;
1523 
1524         /* Locate the Unicast table entry */
1525         last_nibble = (0xf & last_nibble);
1526 
1527         /* offset from unicast tbl base */
1528         tbl_offset = (last_nibble / 4) * 4;
1529 
1530         /* offset within the above reg  */
1531         reg_offset = last_nibble % 4;
1532 
1533         unicast_reg = mvreg_read(pp, (MVNETA_DA_FILT_UCAST_BASE + tbl_offset));
1534 
1535         if (queue == -1) {
1536                 /* Clear accepts frame bit at specified unicast DA tbl entry */
1537                 unicast_reg &= ~(0xff << (8 * reg_offset));
1538         } else {
1539                 unicast_reg &= ~(0xff << (8 * reg_offset));
1540                 unicast_reg |= ((0x01 | (queue << 1)) << (8 * reg_offset));
1541         }
1542 
1543         mvreg_write(pp, (MVNETA_DA_FILT_UCAST_BASE + tbl_offset), unicast_reg);
1544 }
1545 
1546 /* Set mac address */
1547 static void mvneta_mac_addr_set(struct mvneta_port *pp, unsigned char *addr,
1548                                 int queue)
1549 {
1550         unsigned int mac_h;
1551         unsigned int mac_l;
1552 
1553         if (queue != -1) {
1554                 mac_l = (addr[4] << 8) | (addr[5]);
1555                 mac_h = (addr[0] << 24) | (addr[1] << 16) |
1556                         (addr[2] << 8) | (addr[3] << 0);
1557 
1558                 mvreg_write(pp, MVNETA_MAC_ADDR_LOW, mac_l);
1559                 mvreg_write(pp, MVNETA_MAC_ADDR_HIGH, mac_h);
1560         }
1561 
1562         /* Accept frames of this address */
1563         mvneta_set_ucast_addr(pp, addr[5], queue);
1564 }
1565 
1566 /* Set the number of packets that will be received before RX interrupt
1567  * will be generated by HW.
1568  */
1569 static void mvneta_rx_pkts_coal_set(struct mvneta_port *pp,
1570                                     struct mvneta_rx_queue *rxq, u32 value)
1571 {
1572         mvreg_write(pp, MVNETA_RXQ_THRESHOLD_REG(rxq->id),
1573                     value | MVNETA_RXQ_NON_OCCUPIED(0));
1574         rxq->pkts_coal = value;
1575 }
1576 
1577 /* Set the time delay in usec before RX interrupt will be generated by
1578  * HW.
1579  */
1580 static void mvneta_rx_time_coal_set(struct mvneta_port *pp,
1581                                     struct mvneta_rx_queue *rxq, u32 value)
1582 {
1583         u32 val;
1584         unsigned long clk_rate;
1585 
1586         clk_rate = clk_get_rate(pp->clk);
1587         val = (clk_rate / 1000000) * value;
1588 
1589         mvreg_write(pp, MVNETA_RXQ_TIME_COAL_REG(rxq->id), val);
1590         rxq->time_coal = value;
1591 }
1592 
1593 /* Set threshold for TX_DONE pkts coalescing */
1594 static void mvneta_tx_done_pkts_coal_set(struct mvneta_port *pp,
1595                                          struct mvneta_tx_queue *txq, u32 value)
1596 {
1597         u32 val;
1598 
1599         val = mvreg_read(pp, MVNETA_TXQ_SIZE_REG(txq->id));
1600 
1601         val &= ~MVNETA_TXQ_SENT_THRESH_ALL_MASK;
1602         val |= MVNETA_TXQ_SENT_THRESH_MASK(value);
1603 
1604         mvreg_write(pp, MVNETA_TXQ_SIZE_REG(txq->id), val);
1605 
1606         txq->done_pkts_coal = value;
1607 }
1608 
1609 /* Handle rx descriptor fill by setting buf_cookie and buf_phys_addr */
1610 static void mvneta_rx_desc_fill(struct mvneta_rx_desc *rx_desc,
1611                                 u32 phys_addr, void *virt_addr,
1612                                 struct mvneta_rx_queue *rxq)
1613 {
1614         int i;
1615 
1616         rx_desc->buf_phys_addr = phys_addr;
1617         i = rx_desc - rxq->descs;
1618         rxq->buf_virt_addr[i] = virt_addr;
1619 }
1620 
1621 /* Decrement sent descriptors counter */
1622 static void mvneta_txq_sent_desc_dec(struct mvneta_port *pp,
1623                                      struct mvneta_tx_queue *txq,
1624                                      int sent_desc)
1625 {
1626         u32 val;
1627 
1628         /* Only 255 TX descriptors can be updated at once */
1629         while (sent_desc > 0xff) {
1630                 val = 0xff << MVNETA_TXQ_DEC_SENT_SHIFT;
1631                 mvreg_write(pp, MVNETA_TXQ_UPDATE_REG(txq->id), val);
1632                 sent_desc = sent_desc - 0xff;
1633         }
1634 
1635         val = sent_desc << MVNETA_TXQ_DEC_SENT_SHIFT;
1636         mvreg_write(pp, MVNETA_TXQ_UPDATE_REG(txq->id), val);
1637 }
1638 
1639 /* Get number of TX descriptors already sent by HW */
1640 static int mvneta_txq_sent_desc_num_get(struct mvneta_port *pp,
1641                                         struct mvneta_tx_queue *txq)
1642 {
1643         u32 val;
1644         int sent_desc;
1645 
1646         val = mvreg_read(pp, MVNETA_TXQ_STATUS_REG(txq->id));
1647         sent_desc = (val & MVNETA_TXQ_SENT_DESC_MASK) >>
1648                 MVNETA_TXQ_SENT_DESC_SHIFT;
1649 
1650         return sent_desc;
1651 }
1652 
1653 /* Get number of sent descriptors and decrement counter.
1654  *  The number of sent descriptors is returned.
1655  */
1656 static int mvneta_txq_sent_desc_proc(struct mvneta_port *pp,
1657                                      struct mvneta_tx_queue *txq)
1658 {
1659         int sent_desc;
1660 
1661         /* Get number of sent descriptors */
1662         sent_desc = mvneta_txq_sent_desc_num_get(pp, txq);
1663 
1664         /* Decrement sent descriptors counter */
1665         if (sent_desc)
1666                 mvneta_txq_sent_desc_dec(pp, txq, sent_desc);
1667 
1668         return sent_desc;
1669 }
1670 
1671 /* Set TXQ descriptors fields relevant for CSUM calculation */
1672 static u32 mvneta_txq_desc_csum(int l3_offs, int l3_proto,
1673                                 int ip_hdr_len, int l4_proto)
1674 {
1675         u32 command;
1676 
1677         /* Fields: L3_offset, IP_hdrlen, L3_type, G_IPv4_chk,
1678          * G_L4_chk, L4_type; required only for checksum
1679          * calculation
1680          */
1681         command =  l3_offs    << MVNETA_TX_L3_OFF_SHIFT;
1682         command |= ip_hdr_len << MVNETA_TX_IP_HLEN_SHIFT;
1683 
1684         if (l3_proto == htons(ETH_P_IP))
1685                 command |= MVNETA_TXD_IP_CSUM;
1686         else
1687                 command |= MVNETA_TX_L3_IP6;
1688 
1689         if (l4_proto == IPPROTO_TCP)
1690                 command |=  MVNETA_TX_L4_CSUM_FULL;
1691         else if (l4_proto == IPPROTO_UDP)
1692                 command |= MVNETA_TX_L4_UDP | MVNETA_TX_L4_CSUM_FULL;
1693         else
1694                 command |= MVNETA_TX_L4_CSUM_NOT;
1695 
1696         return command;
1697 }
1698 
1699 
1700 /* Display more error info */
1701 static void mvneta_rx_error(struct mvneta_port *pp,
1702                             struct mvneta_rx_desc *rx_desc)
1703 {
1704         u32 status = rx_desc->status;
1705 
1706         if (!mvneta_rxq_desc_is_first_last(status)) {
1707                 netdev_err(pp->dev,
1708                            "bad rx status %08x (buffer oversize), size=%d\n",
1709                            status, rx_desc->data_size);
1710                 return;
1711         }
1712 
1713         switch (status & MVNETA_RXD_ERR_CODE_MASK) {
1714         case MVNETA_RXD_ERR_CRC:
1715                 netdev_err(pp->dev, "bad rx status %08x (crc error), size=%d\n",
1716                            status, rx_desc->data_size);
1717                 break;
1718         case MVNETA_RXD_ERR_OVERRUN:
1719                 netdev_err(pp->dev, "bad rx status %08x (overrun error), size=%d\n",
1720                            status, rx_desc->data_size);
1721                 break;
1722         case MVNETA_RXD_ERR_LEN:
1723                 netdev_err(pp->dev, "bad rx status %08x (max frame length error), size=%d\n",
1724                            status, rx_desc->data_size);
1725                 break;
1726         case MVNETA_RXD_ERR_RESOURCE:
1727                 netdev_err(pp->dev, "bad rx status %08x (resource error), size=%d\n",
1728                            status, rx_desc->data_size);
1729                 break;
1730         }
1731 }
1732 
1733 /* Handle RX checksum offload based on the descriptor's status */
1734 static void mvneta_rx_csum(struct mvneta_port *pp, u32 status,
1735                            struct sk_buff *skb)
1736 {
1737         if ((status & MVNETA_RXD_L3_IP4) &&
1738             (status & MVNETA_RXD_L4_CSUM_OK)) {
1739                 skb->csum = 0;
1740                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1741                 return;
1742         }
1743 
1744         skb->ip_summed = CHECKSUM_NONE;
1745 }
1746 
1747 /* Return tx queue pointer (find last set bit) according to <cause> returned
1748  * form tx_done reg. <cause> must not be null. The return value is always a
1749  * valid queue for matching the first one found in <cause>.
1750  */
1751 static struct mvneta_tx_queue *mvneta_tx_done_policy(struct mvneta_port *pp,
1752                                                      u32 cause)
1753 {
1754         int queue = fls(cause) - 1;
1755 
1756         return &pp->txqs[queue];
1757 }
1758 
1759 /* Free tx queue skbuffs */
1760 static void mvneta_txq_bufs_free(struct mvneta_port *pp,
1761                                  struct mvneta_tx_queue *txq, int num)
1762 {
1763         int i;
1764 
1765         for (i = 0; i < num; i++) {
1766                 struct mvneta_tx_desc *tx_desc = txq->descs +
1767                         txq->txq_get_index;
1768                 struct sk_buff *skb = txq->tx_skb[txq->txq_get_index];
1769 
1770                 mvneta_txq_inc_get(txq);
1771 
1772                 if (!IS_TSO_HEADER(txq, tx_desc->buf_phys_addr))
1773                         dma_unmap_single(pp->dev->dev.parent,
1774                                          tx_desc->buf_phys_addr,
1775                                          tx_desc->data_size, DMA_TO_DEVICE);
1776                 if (!skb)
1777                         continue;
1778                 dev_kfree_skb_any(skb);
1779         }
1780 }
1781 
1782 /* Handle end of transmission */
1783 static void mvneta_txq_done(struct mvneta_port *pp,
1784                            struct mvneta_tx_queue *txq)
1785 {
1786         struct netdev_queue *nq = netdev_get_tx_queue(pp->dev, txq->id);
1787         int tx_done;
1788 
1789         tx_done = mvneta_txq_sent_desc_proc(pp, txq);
1790         if (!tx_done)
1791                 return;
1792 
1793         mvneta_txq_bufs_free(pp, txq, tx_done);
1794 
1795         txq->count -= tx_done;
1796 
1797         if (netif_tx_queue_stopped(nq)) {
1798                 if (txq->count <= txq->tx_wake_threshold)
1799                         netif_tx_wake_queue(nq);
1800         }
1801 }
1802 
1803 void *mvneta_frag_alloc(unsigned int frag_size)
1804 {
1805         if (likely(frag_size <= PAGE_SIZE))
1806                 return netdev_alloc_frag(frag_size);
1807         else
1808                 return kmalloc(frag_size, GFP_ATOMIC);
1809 }
1810 EXPORT_SYMBOL_GPL(mvneta_frag_alloc);
1811 
1812 void mvneta_frag_free(unsigned int frag_size, void *data)
1813 {
1814         if (likely(frag_size <= PAGE_SIZE))
1815                 skb_free_frag(data);
1816         else
1817                 kfree(data);
1818 }
1819 EXPORT_SYMBOL_GPL(mvneta_frag_free);
1820 
1821 /* Refill processing for SW buffer management */
1822 static int mvneta_rx_refill(struct mvneta_port *pp,
1823                             struct mvneta_rx_desc *rx_desc,
1824                             struct mvneta_rx_queue *rxq)
1825 
1826 {
1827         dma_addr_t phys_addr;
1828         void *data;
1829 
1830         data = mvneta_frag_alloc(pp->frag_size);
1831         if (!data)
1832                 return -ENOMEM;
1833 
1834         phys_addr = dma_map_single(pp->dev->dev.parent, data,
1835                                    MVNETA_RX_BUF_SIZE(pp->pkt_size),
1836                                    DMA_FROM_DEVICE);
1837         if (unlikely(dma_mapping_error(pp->dev->dev.parent, phys_addr))) {
1838                 mvneta_frag_free(pp->frag_size, data);
1839                 return -ENOMEM;
1840         }
1841 
1842         phys_addr += pp->rx_offset_correction;
1843         mvneta_rx_desc_fill(rx_desc, phys_addr, data, rxq);
1844         return 0;
1845 }
1846 
1847 /* Handle tx checksum */
1848 static u32 mvneta_skb_tx_csum(struct mvneta_port *pp, struct sk_buff *skb)
1849 {
1850         if (skb->ip_summed == CHECKSUM_PARTIAL) {
1851                 int ip_hdr_len = 0;
1852                 __be16 l3_proto = vlan_get_protocol(skb);
1853                 u8 l4_proto;
1854 
1855                 if (l3_proto == htons(ETH_P_IP)) {
1856                         struct iphdr *ip4h = ip_hdr(skb);
1857 
1858                         /* Calculate IPv4 checksum and L4 checksum */
1859                         ip_hdr_len = ip4h->ihl;
1860                         l4_proto = ip4h->protocol;
1861                 } else if (l3_proto == htons(ETH_P_IPV6)) {
1862                         struct ipv6hdr *ip6h = ipv6_hdr(skb);
1863 
1864                         /* Read l4_protocol from one of IPv6 extra headers */
1865                         if (skb_network_header_len(skb) > 0)
1866                                 ip_hdr_len = (skb_network_header_len(skb) >> 2);
1867                         l4_proto = ip6h->nexthdr;
1868                 } else
1869                         return MVNETA_TX_L4_CSUM_NOT;
1870 
1871                 return mvneta_txq_desc_csum(skb_network_offset(skb),
1872                                             l3_proto, ip_hdr_len, l4_proto);
1873         }
1874 
1875         return MVNETA_TX_L4_CSUM_NOT;
1876 }
1877 
1878 /* Drop packets received by the RXQ and free buffers */
1879 static void mvneta_rxq_drop_pkts(struct mvneta_port *pp,
1880                                  struct mvneta_rx_queue *rxq)
1881 {
1882         int rx_done, i;
1883 
1884         rx_done = mvneta_rxq_busy_desc_num_get(pp, rxq);
1885         if (rx_done)
1886                 mvneta_rxq_desc_num_update(pp, rxq, rx_done, rx_done);
1887 
1888         if (pp->bm_priv) {
1889                 for (i = 0; i < rx_done; i++) {
1890                         struct mvneta_rx_desc *rx_desc =
1891                                                   mvneta_rxq_next_desc_get(rxq);
1892                         u8 pool_id = MVNETA_RX_GET_BM_POOL_ID(rx_desc);
1893                         struct mvneta_bm_pool *bm_pool;
1894 
1895                         bm_pool = &pp->bm_priv->bm_pools[pool_id];
1896                         /* Return dropped buffer to the pool */
1897                         mvneta_bm_pool_put_bp(pp->bm_priv, bm_pool,
1898                                               rx_desc->buf_phys_addr);
1899                 }
1900                 return;
1901         }
1902 
1903         for (i = 0; i < rxq->size; i++) {
1904                 struct mvneta_rx_desc *rx_desc = rxq->descs + i;
1905                 void *data = rxq->buf_virt_addr[i];
1906 
1907                 dma_unmap_single(pp->dev->dev.parent, rx_desc->buf_phys_addr,
1908                                  MVNETA_RX_BUF_SIZE(pp->pkt_size), DMA_FROM_DEVICE);
1909                 mvneta_frag_free(pp->frag_size, data);
1910         }
1911 }
1912 
1913 /* Main rx processing when using software buffer management */
1914 static int mvneta_rx_swbm(struct mvneta_port *pp, int rx_todo,
1915                           struct mvneta_rx_queue *rxq)
1916 {
1917         struct mvneta_pcpu_port *port = this_cpu_ptr(pp->ports);
1918         struct net_device *dev = pp->dev;
1919         int rx_done;
1920         u32 rcvd_pkts = 0;
1921         u32 rcvd_bytes = 0;
1922 
1923         /* Get number of received packets */
1924         rx_done = mvneta_rxq_busy_desc_num_get(pp, rxq);
1925 
1926         if (rx_todo > rx_done)
1927                 rx_todo = rx_done;
1928 
1929         rx_done = 0;
1930 
1931         /* Fairness NAPI loop */
1932         while (rx_done < rx_todo) {
1933                 struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq);
1934                 struct sk_buff *skb;
1935                 unsigned char *data;
1936                 dma_addr_t phys_addr;
1937                 u32 rx_status, frag_size;
1938                 int rx_bytes, err, index;
1939 
1940                 rx_done++;
1941                 rx_status = rx_desc->status;
1942                 rx_bytes = rx_desc->data_size - (ETH_FCS_LEN + MVNETA_MH_SIZE);
1943                 index = rx_desc - rxq->descs;
1944                 data = rxq->buf_virt_addr[index];
1945                 phys_addr = rx_desc->buf_phys_addr;
1946 
1947                 if (!mvneta_rxq_desc_is_first_last(rx_status) ||
1948                     (rx_status & MVNETA_RXD_ERR_SUMMARY)) {
1949 err_drop_frame:
1950                         dev->stats.rx_errors++;
1951                         mvneta_rx_error(pp, rx_desc);
1952                         /* leave the descriptor untouched */
1953                         continue;
1954                 }
1955 
1956                 if (rx_bytes <= rx_copybreak) {
1957                 /* better copy a small frame and not unmap the DMA region */
1958                         skb = netdev_alloc_skb_ip_align(dev, rx_bytes);
1959                         if (unlikely(!skb))
1960                                 goto err_drop_frame;
1961 
1962                         dma_sync_single_range_for_cpu(dev->dev.parent,
1963                                                       phys_addr,
1964                                                       MVNETA_MH_SIZE + NET_SKB_PAD,
1965                                                       rx_bytes,
1966                                                       DMA_FROM_DEVICE);
1967                         memcpy(skb_put(skb, rx_bytes),
1968                                data + MVNETA_MH_SIZE + NET_SKB_PAD,
1969                                rx_bytes);
1970 
1971                         skb->protocol = eth_type_trans(skb, dev);
1972                         mvneta_rx_csum(pp, rx_status, skb);
1973                         napi_gro_receive(&port->napi, skb);
1974 
1975                         rcvd_pkts++;
1976                         rcvd_bytes += rx_bytes;
1977 
1978                         /* leave the descriptor and buffer untouched */
1979                         continue;
1980                 }
1981 
1982                 /* Refill processing */
1983                 err = mvneta_rx_refill(pp, rx_desc, rxq);
1984                 if (err) {
1985                         netdev_err(dev, "Linux processing - Can't refill\n");
1986                         rxq->missed++;
1987                         goto err_drop_frame;
1988                 }
1989 
1990                 frag_size = pp->frag_size;
1991 
1992                 skb = build_skb(data, frag_size > PAGE_SIZE ? 0 : frag_size);
1993 
1994                 /* After refill old buffer has to be unmapped regardless
1995                  * the skb is successfully built or not.
1996                  */
1997                 dma_unmap_single(dev->dev.parent, phys_addr,
1998                                  MVNETA_RX_BUF_SIZE(pp->pkt_size),
1999                                  DMA_FROM_DEVICE);
2000 
2001                 if (!skb)
2002                         goto err_drop_frame;
2003 
2004                 rcvd_pkts++;
2005                 rcvd_bytes += rx_bytes;
2006 
2007                 /* Linux processing */
2008                 skb_reserve(skb, MVNETA_MH_SIZE + NET_SKB_PAD);
2009                 skb_put(skb, rx_bytes);
2010 
2011                 skb->protocol = eth_type_trans(skb, dev);
2012 
2013                 mvneta_rx_csum(pp, rx_status, skb);
2014 
2015                 napi_gro_receive(&port->napi, skb);
2016         }
2017 
2018         if (rcvd_pkts) {
2019                 struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
2020 
2021                 u64_stats_update_begin(&stats->syncp);
2022                 stats->rx_packets += rcvd_pkts;
2023                 stats->rx_bytes   += rcvd_bytes;
2024                 u64_stats_update_end(&stats->syncp);
2025         }
2026 
2027         /* Update rxq management counters */
2028         mvneta_rxq_desc_num_update(pp, rxq, rx_done, rx_done);
2029 
2030         return rx_done;
2031 }
2032 
2033 /* Main rx processing when using hardware buffer management */
2034 static int mvneta_rx_hwbm(struct mvneta_port *pp, int rx_todo,
2035                           struct mvneta_rx_queue *rxq)
2036 {
2037         struct mvneta_pcpu_port *port = this_cpu_ptr(pp->ports);
2038         struct net_device *dev = pp->dev;
2039         int rx_done;
2040         u32 rcvd_pkts = 0;
2041         u32 rcvd_bytes = 0;
2042 
2043         /* Get number of received packets */
2044         rx_done = mvneta_rxq_busy_desc_num_get(pp, rxq);
2045 
2046         if (rx_todo > rx_done)
2047                 rx_todo = rx_done;
2048 
2049         rx_done = 0;
2050 
2051         /* Fairness NAPI loop */
2052         while (rx_done < rx_todo) {
2053                 struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq);
2054                 struct mvneta_bm_pool *bm_pool = NULL;
2055                 struct sk_buff *skb;
2056                 unsigned char *data;
2057                 dma_addr_t phys_addr;
2058                 u32 rx_status, frag_size;
2059                 int rx_bytes, err;
2060                 u8 pool_id;
2061 
2062                 rx_done++;
2063                 rx_status = rx_desc->status;
2064                 rx_bytes = rx_desc->data_size - (ETH_FCS_LEN + MVNETA_MH_SIZE);
2065                 data = (u8 *)(uintptr_t)rx_desc->buf_cookie;
2066                 phys_addr = rx_desc->buf_phys_addr;
2067                 pool_id = MVNETA_RX_GET_BM_POOL_ID(rx_desc);
2068                 bm_pool = &pp->bm_priv->bm_pools[pool_id];
2069 
2070                 if (!mvneta_rxq_desc_is_first_last(rx_status) ||
2071                     (rx_status & MVNETA_RXD_ERR_SUMMARY)) {
2072 err_drop_frame_ret_pool:
2073                         /* Return the buffer to the pool */
2074                         mvneta_bm_pool_put_bp(pp->bm_priv, bm_pool,
2075                                               rx_desc->buf_phys_addr);
2076 err_drop_frame:
2077                         dev->stats.rx_errors++;
2078                         mvneta_rx_error(pp, rx_desc);
2079                         /* leave the descriptor untouched */
2080                         continue;
2081                 }
2082 
2083                 if (rx_bytes <= rx_copybreak) {
2084                         /* better copy a small frame and not unmap the DMA region */
2085                         skb = netdev_alloc_skb_ip_align(dev, rx_bytes);
2086                         if (unlikely(!skb))
2087                                 goto err_drop_frame_ret_pool;
2088 
2089                         dma_sync_single_range_for_cpu(dev->dev.parent,
2090                                                       rx_desc->buf_phys_addr,
2091                                                       MVNETA_MH_SIZE + NET_SKB_PAD,
2092                                                       rx_bytes,
2093                                                       DMA_FROM_DEVICE);
2094                         memcpy(skb_put(skb, rx_bytes),
2095                                data + MVNETA_MH_SIZE + NET_SKB_PAD,
2096                                rx_bytes);
2097 
2098                         skb->protocol = eth_type_trans(skb, dev);
2099                         mvneta_rx_csum(pp, rx_status, skb);
2100                         napi_gro_receive(&port->napi, skb);
2101 
2102                         rcvd_pkts++;
2103                         rcvd_bytes += rx_bytes;
2104 
2105                         /* Return the buffer to the pool */
2106                         mvneta_bm_pool_put_bp(pp->bm_priv, bm_pool,
2107                                               rx_desc->buf_phys_addr);
2108 
2109                         /* leave the descriptor and buffer untouched */
2110                         continue;
2111                 }
2112 
2113                 /* Refill processing */
2114                 err = hwbm_pool_refill(&bm_pool->hwbm_pool, GFP_ATOMIC);
2115                 if (err) {
2116                         netdev_err(dev, "Linux processing - Can't refill\n");
2117                         rxq->missed++;
2118                         goto err_drop_frame_ret_pool;
2119                 }
2120 
2121                 frag_size = bm_pool->hwbm_pool.frag_size;
2122 
2123                 skb = build_skb(data, frag_size > PAGE_SIZE ? 0 : frag_size);
2124 
2125                 /* After refill old buffer has to be unmapped regardless
2126                  * the skb is successfully built or not.
2127                  */
2128                 dma_unmap_single(&pp->bm_priv->pdev->dev, phys_addr,
2129                                  bm_pool->buf_size, DMA_FROM_DEVICE);
2130                 if (!skb)
2131                         goto err_drop_frame;
2132 
2133                 rcvd_pkts++;
2134                 rcvd_bytes += rx_bytes;
2135 
2136                 /* Linux processing */
2137                 skb_reserve(skb, MVNETA_MH_SIZE + NET_SKB_PAD);
2138                 skb_put(skb, rx_bytes);
2139 
2140                 skb->protocol = eth_type_trans(skb, dev);
2141 
2142                 mvneta_rx_csum(pp, rx_status, skb);
2143 
2144                 napi_gro_receive(&port->napi, skb);
2145         }
2146 
2147         if (rcvd_pkts) {
2148                 struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
2149 
2150                 u64_stats_update_begin(&stats->syncp);
2151                 stats->rx_packets += rcvd_pkts;
2152                 stats->rx_bytes   += rcvd_bytes;
2153                 u64_stats_update_end(&stats->syncp);
2154         }
2155 
2156         /* Update rxq management counters */
2157         mvneta_rxq_desc_num_update(pp, rxq, rx_done, rx_done);
2158 
2159         return rx_done;
2160 }
2161 
2162 static inline void
2163 mvneta_tso_put_hdr(struct sk_buff *skb,
2164                    struct mvneta_port *pp, struct mvneta_tx_queue *txq)
2165 {
2166         struct mvneta_tx_desc *tx_desc;
2167         int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2168 
2169         txq->tx_skb[txq->txq_put_index] = NULL;
2170         tx_desc = mvneta_txq_next_desc_get(txq);
2171         tx_desc->data_size = hdr_len;
2172         tx_desc->command = mvneta_skb_tx_csum(pp, skb);
2173         tx_desc->command |= MVNETA_TXD_F_DESC;
2174         tx_desc->buf_phys_addr = txq->tso_hdrs_phys +
2175                                  txq->txq_put_index * TSO_HEADER_SIZE;
2176         mvneta_txq_inc_put(txq);
2177 }
2178 
2179 static inline int
2180 mvneta_tso_put_data(struct net_device *dev, struct mvneta_tx_queue *txq,
2181                     struct sk_buff *skb, char *data, int size,
2182                     bool last_tcp, bool is_last)
2183 {
2184         struct mvneta_tx_desc *tx_desc;
2185 
2186         tx_desc = mvneta_txq_next_desc_get(txq);
2187         tx_desc->data_size = size;
2188         tx_desc->buf_phys_addr = dma_map_single(dev->dev.parent, data,
2189                                                 size, DMA_TO_DEVICE);
2190         if (unlikely(dma_mapping_error(dev->dev.parent,
2191                      tx_desc->buf_phys_addr))) {
2192                 mvneta_txq_desc_put(txq);
2193                 return -ENOMEM;
2194         }
2195 
2196         tx_desc->command = 0;
2197         txq->tx_skb[txq->txq_put_index] = NULL;
2198 
2199         if (last_tcp) {
2200                 /* last descriptor in the TCP packet */
2201                 tx_desc->command = MVNETA_TXD_L_DESC;
2202 
2203                 /* last descriptor in SKB */
2204                 if (is_last)
2205                         txq->tx_skb[txq->txq_put_index] = skb;
2206         }
2207         mvneta_txq_inc_put(txq);
2208         return 0;
2209 }
2210 
2211 static int mvneta_tx_tso(struct sk_buff *skb, struct net_device *dev,
2212                          struct mvneta_tx_queue *txq)
2213 {
2214         int total_len, data_left;
2215         int desc_count = 0;
2216         struct mvneta_port *pp = netdev_priv(dev);
2217         struct tso_t tso;
2218         int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2219         int i;
2220 
2221         /* Count needed descriptors */
2222         if ((txq->count + tso_count_descs(skb)) >= txq->size)
2223                 return 0;
2224 
2225         if (skb_headlen(skb) < (skb_transport_offset(skb) + tcp_hdrlen(skb))) {
2226                 pr_info("*** Is this even  possible???!?!?\n");
2227                 return 0;
2228         }
2229 
2230         /* Initialize the TSO handler, and prepare the first payload */
2231         tso_start(skb, &tso);
2232 
2233         total_len = skb->len - hdr_len;
2234         while (total_len > 0) {
2235                 char *hdr;
2236 
2237                 data_left = min_t(int, skb_shinfo(skb)->gso_size, total_len);
2238                 total_len -= data_left;
2239                 desc_count++;
2240 
2241                 /* prepare packet headers: MAC + IP + TCP */
2242                 hdr = txq->tso_hdrs + txq->txq_put_index * TSO_HEADER_SIZE;
2243                 tso_build_hdr(skb, hdr, &tso, data_left, total_len == 0);
2244 
2245                 mvneta_tso_put_hdr(skb, pp, txq);
2246 
2247                 while (data_left > 0) {
2248                         int size;
2249                         desc_count++;
2250 
2251                         size = min_t(int, tso.size, data_left);
2252 
2253                         if (mvneta_tso_put_data(dev, txq, skb,
2254                                                  tso.data, size,
2255                                                  size == data_left,
2256                                                  total_len == 0))
2257                                 goto err_release;
2258                         data_left -= size;
2259 
2260                         tso_build_data(skb, &tso, size);
2261                 }
2262         }
2263 
2264         return desc_count;
2265 
2266 err_release:
2267         /* Release all used data descriptors; header descriptors must not
2268          * be DMA-unmapped.
2269          */
2270         for (i = desc_count - 1; i >= 0; i--) {
2271                 struct mvneta_tx_desc *tx_desc = txq->descs + i;
2272                 if (!IS_TSO_HEADER(txq, tx_desc->buf_phys_addr))
2273                         dma_unmap_single(pp->dev->dev.parent,
2274                                          tx_desc->buf_phys_addr,
2275                                          tx_desc->data_size,
2276                                          DMA_TO_DEVICE);
2277                 mvneta_txq_desc_put(txq);
2278         }
2279         return 0;
2280 }
2281 
2282 /* Handle tx fragmentation processing */
2283 static int mvneta_tx_frag_process(struct mvneta_port *pp, struct sk_buff *skb,
2284                                   struct mvneta_tx_queue *txq)
2285 {
2286         struct mvneta_tx_desc *tx_desc;
2287         int i, nr_frags = skb_shinfo(skb)->nr_frags;
2288 
2289         for (i = 0; i < nr_frags; i++) {
2290                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2291                 void *addr = page_address(frag->page.p) + frag->page_offset;
2292 
2293                 tx_desc = mvneta_txq_next_desc_get(txq);
2294                 tx_desc->data_size = frag->size;
2295 
2296                 tx_desc->buf_phys_addr =
2297                         dma_map_single(pp->dev->dev.parent, addr,
2298                                        tx_desc->data_size, DMA_TO_DEVICE);
2299 
2300                 if (dma_mapping_error(pp->dev->dev.parent,
2301                                       tx_desc->buf_phys_addr)) {
2302                         mvneta_txq_desc_put(txq);
2303                         goto error;
2304                 }
2305 
2306                 if (i == nr_frags - 1) {
2307                         /* Last descriptor */
2308                         tx_desc->command = MVNETA_TXD_L_DESC | MVNETA_TXD_Z_PAD;
2309                         txq->tx_skb[txq->txq_put_index] = skb;
2310                 } else {
2311                         /* Descriptor in the middle: Not First, Not Last */
2312                         tx_desc->command = 0;
2313                         txq->tx_skb[txq->txq_put_index] = NULL;
2314                 }
2315                 mvneta_txq_inc_put(txq);
2316         }
2317 
2318         return 0;
2319 
2320 error:
2321         /* Release all descriptors that were used to map fragments of
2322          * this packet, as well as the corresponding DMA mappings
2323          */
2324         for (i = i - 1; i >= 0; i--) {
2325                 tx_desc = txq->descs + i;
2326                 dma_unmap_single(pp->dev->dev.parent,
2327                                  tx_desc->buf_phys_addr,
2328                                  tx_desc->data_size,
2329                                  DMA_TO_DEVICE);
2330                 mvneta_txq_desc_put(txq);
2331         }
2332 
2333         return -ENOMEM;
2334 }
2335 
2336 /* Main tx processing */
2337 static int mvneta_tx(struct sk_buff *skb, struct net_device *dev)
2338 {
2339         struct mvneta_port *pp = netdev_priv(dev);
2340         u16 txq_id = skb_get_queue_mapping(skb);
2341         struct mvneta_tx_queue *txq = &pp->txqs[txq_id];
2342         struct mvneta_tx_desc *tx_desc;
2343         int len = skb->len;
2344         int frags = 0;
2345         u32 tx_cmd;
2346 
2347         if (!netif_running(dev))
2348                 goto out;
2349 
2350         if (skb_is_gso(skb)) {
2351                 frags = mvneta_tx_tso(skb, dev, txq);
2352                 goto out;
2353         }
2354 
2355         frags = skb_shinfo(skb)->nr_frags + 1;
2356 
2357         /* Get a descriptor for the first part of the packet */
2358         tx_desc = mvneta_txq_next_desc_get(txq);
2359 
2360         tx_cmd = mvneta_skb_tx_csum(pp, skb);
2361 
2362         tx_desc->data_size = skb_headlen(skb);
2363 
2364         tx_desc->buf_phys_addr = dma_map_single(dev->dev.parent, skb->data,
2365                                                 tx_desc->data_size,
2366                                                 DMA_TO_DEVICE);
2367         if (unlikely(dma_mapping_error(dev->dev.parent,
2368                                        tx_desc->buf_phys_addr))) {
2369                 mvneta_txq_desc_put(txq);
2370                 frags = 0;
2371                 goto out;
2372         }
2373 
2374         if (frags == 1) {
2375                 /* First and Last descriptor */
2376                 tx_cmd |= MVNETA_TXD_FLZ_DESC;
2377                 tx_desc->command = tx_cmd;
2378                 txq->tx_skb[txq->txq_put_index] = skb;
2379                 mvneta_txq_inc_put(txq);
2380         } else {
2381                 /* First but not Last */
2382                 tx_cmd |= MVNETA_TXD_F_DESC;
2383                 txq->tx_skb[txq->txq_put_index] = NULL;
2384                 mvneta_txq_inc_put(txq);
2385                 tx_desc->command = tx_cmd;
2386                 /* Continue with other skb fragments */
2387                 if (mvneta_tx_frag_process(pp, skb, txq)) {
2388                         dma_unmap_single(dev->dev.parent,
2389                                          tx_desc->buf_phys_addr,
2390                                          tx_desc->data_size,
2391                                          DMA_TO_DEVICE);
2392                         mvneta_txq_desc_put(txq);
2393                         frags = 0;
2394                         goto out;
2395                 }
2396         }
2397 
2398 out:
2399         if (frags > 0) {
2400                 struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
2401                 struct netdev_queue *nq = netdev_get_tx_queue(dev, txq_id);
2402 
2403                 txq->count += frags;
2404                 mvneta_txq_pend_desc_add(pp, txq, frags);
2405 
2406                 if (txq->count >= txq->tx_stop_threshold)
2407                         netif_tx_stop_queue(nq);
2408 
2409                 u64_stats_update_begin(&stats->syncp);
2410                 stats->tx_packets++;
2411                 stats->tx_bytes  += len;
2412                 u64_stats_update_end(&stats->syncp);
2413         } else {
2414                 dev->stats.tx_dropped++;
2415                 dev_kfree_skb_any(skb);
2416         }
2417 
2418         return NETDEV_TX_OK;
2419 }
2420 
2421 
2422 /* Free tx resources, when resetting a port */
2423 static void mvneta_txq_done_force(struct mvneta_port *pp,
2424                                   struct mvneta_tx_queue *txq)
2425 
2426 {
2427         int tx_done = txq->count;
2428 
2429         mvneta_txq_bufs_free(pp, txq, tx_done);
2430 
2431         /* reset txq */
2432         txq->count = 0;
2433         txq->txq_put_index = 0;
2434         txq->txq_get_index = 0;
2435 }
2436 
2437 /* Handle tx done - called in softirq context. The <cause_tx_done> argument
2438  * must be a valid cause according to MVNETA_TXQ_INTR_MASK_ALL.
2439  */
2440 static void mvneta_tx_done_gbe(struct mvneta_port *pp, u32 cause_tx_done)
2441 {
2442         struct mvneta_tx_queue *txq;
2443         struct netdev_queue *nq;
2444 
2445         while (cause_tx_done) {
2446                 txq = mvneta_tx_done_policy(pp, cause_tx_done);
2447 
2448                 nq = netdev_get_tx_queue(pp->dev, txq->id);
2449                 __netif_tx_lock(nq, smp_processor_id());
2450 
2451                 if (txq->count)
2452                         mvneta_txq_done(pp, txq);
2453 
2454                 __netif_tx_unlock(nq);
2455                 cause_tx_done &= ~((1 << txq->id));
2456         }
2457 }
2458 
2459 /* Compute crc8 of the specified address, using a unique algorithm ,
2460  * according to hw spec, different than generic crc8 algorithm
2461  */
2462 static int mvneta_addr_crc(unsigned char *addr)
2463 {
2464         int crc = 0;
2465         int i;
2466 
2467         for (i = 0; i < ETH_ALEN; i++) {
2468                 int j;
2469 
2470                 crc = (crc ^ addr[i]) << 8;
2471                 for (j = 7; j >= 0; j--) {
2472                         if (crc & (0x100 << j))
2473                                 crc ^= 0x107 << j;
2474                 }
2475         }
2476 
2477         return crc;
2478 }
2479 
2480 /* This method controls the net device special MAC multicast support.
2481  * The Special Multicast Table for MAC addresses supports MAC of the form
2482  * 0x01-00-5E-00-00-XX (where XX is between 0x00 and 0xFF).
2483  * The MAC DA[7:0] bits are used as a pointer to the Special Multicast
2484  * Table entries in the DA-Filter table. This method set the Special
2485  * Multicast Table appropriate entry.
2486  */
2487 static void mvneta_set_special_mcast_addr(struct mvneta_port *pp,
2488                                           unsigned char last_byte,
2489                                           int queue)
2490 {
2491         unsigned int smc_table_reg;
2492         unsigned int tbl_offset;
2493         unsigned int reg_offset;
2494 
2495         /* Register offset from SMC table base    */
2496         tbl_offset = (last_byte / 4);
2497         /* Entry offset within the above reg */
2498         reg_offset = last_byte % 4;
2499 
2500         smc_table_reg = mvreg_read(pp, (MVNETA_DA_FILT_SPEC_MCAST
2501                                         + tbl_offset * 4));
2502 
2503         if (queue == -1)
2504                 smc_table_reg &= ~(0xff << (8 * reg_offset));
2505         else {
2506                 smc_table_reg &= ~(0xff << (8 * reg_offset));
2507                 smc_table_reg |= ((0x01 | (queue << 1)) << (8 * reg_offset));
2508         }
2509 
2510         mvreg_write(pp, MVNETA_DA_FILT_SPEC_MCAST + tbl_offset * 4,
2511                     smc_table_reg);
2512 }
2513 
2514 /* This method controls the network device Other MAC multicast support.
2515  * The Other Multicast Table is used for multicast of another type.
2516  * A CRC-8 is used as an index to the Other Multicast Table entries
2517  * in the DA-Filter table.
2518  * The method gets the CRC-8 value from the calling routine and
2519  * sets the Other Multicast Table appropriate entry according to the
2520  * specified CRC-8 .
2521  */
2522 static void mvneta_set_other_mcast_addr(struct mvneta_port *pp,
2523                                         unsigned char crc8,
2524                                         int queue)
2525 {
2526         unsigned int omc_table_reg;
2527         unsigned int tbl_offset;
2528         unsigned int reg_offset;
2529 
2530         tbl_offset = (crc8 / 4) * 4; /* Register offset from OMC table base */
2531         reg_offset = crc8 % 4;       /* Entry offset within the above reg   */
2532 
2533         omc_table_reg = mvreg_read(pp, MVNETA_DA_FILT_OTH_MCAST + tbl_offset);
2534 
2535         if (queue == -1) {
2536                 /* Clear accepts frame bit at specified Other DA table entry */
2537                 omc_table_reg &= ~(0xff << (8 * reg_offset));
2538         } else {
2539                 omc_table_reg &= ~(0xff << (8 * reg_offset));
2540                 omc_table_reg |= ((0x01 | (queue << 1)) << (8 * reg_offset));
2541         }
2542 
2543         mvreg_write(pp, MVNETA_DA_FILT_OTH_MCAST + tbl_offset, omc_table_reg);
2544 }
2545 
2546 /* The network device supports multicast using two tables:
2547  *    1) Special Multicast Table for MAC addresses of the form
2548  *       0x01-00-5E-00-00-XX (where XX is between 0x00 and 0xFF).
2549  *       The MAC DA[7:0] bits are used as a pointer to the Special Multicast
2550  *       Table entries in the DA-Filter table.
2551  *    2) Other Multicast Table for multicast of another type. A CRC-8 value
2552  *       is used as an index to the Other Multicast Table entries in the
2553  *       DA-Filter table.
2554  */
2555 static int mvneta_mcast_addr_set(struct mvneta_port *pp, unsigned char *p_addr,
2556                                  int queue)
2557 {
2558         unsigned char crc_result = 0;
2559 
2560         if (memcmp(p_addr, "\x01\x00\x5e\x00\x00", 5) == 0) {
2561                 mvneta_set_special_mcast_addr(pp, p_addr[5], queue);
2562                 return 0;
2563         }
2564 
2565         crc_result = mvneta_addr_crc(p_addr);
2566         if (queue == -1) {
2567                 if (pp->mcast_count[crc_result] == 0) {
2568                         netdev_info(pp->dev, "No valid Mcast for crc8=0x%02x\n",
2569                                     crc_result);
2570                         return -EINVAL;
2571                 }
2572 
2573                 pp->mcast_count[crc_result]--;
2574                 if (pp->mcast_count[crc_result] != 0) {
2575                         netdev_info(pp->dev,
2576                                     "After delete there are %d valid Mcast for crc8=0x%02x\n",
2577                                     pp->mcast_count[crc_result], crc_result);
2578                         return -EINVAL;
2579                 }
2580         } else
2581                 pp->mcast_count[crc_result]++;
2582 
2583         mvneta_set_other_mcast_addr(pp, crc_result, queue);
2584 
2585         return 0;
2586 }
2587 
2588 /* Configure Fitering mode of Ethernet port */
2589 static void mvneta_rx_unicast_promisc_set(struct mvneta_port *pp,
2590                                           int is_promisc)
2591 {
2592         u32 port_cfg_reg, val;
2593 
2594         port_cfg_reg = mvreg_read(pp, MVNETA_PORT_CONFIG);
2595 
2596         val = mvreg_read(pp, MVNETA_TYPE_PRIO);
2597 
2598         /* Set / Clear UPM bit in port configuration register */
2599         if (is_promisc) {
2600                 /* Accept all Unicast addresses */
2601                 port_cfg_reg |= MVNETA_UNI_PROMISC_MODE;
2602                 val |= MVNETA_FORCE_UNI;
2603                 mvreg_write(pp, MVNETA_MAC_ADDR_LOW, 0xffff);
2604                 mvreg_write(pp, MVNETA_MAC_ADDR_HIGH, 0xffffffff);
2605         } else {
2606                 /* Reject all Unicast addresses */
2607                 port_cfg_reg &= ~MVNETA_UNI_PROMISC_MODE;
2608                 val &= ~MVNETA_FORCE_UNI;
2609         }
2610 
2611         mvreg_write(pp, MVNETA_PORT_CONFIG, port_cfg_reg);
2612         mvreg_write(pp, MVNETA_TYPE_PRIO, val);
2613 }
2614 
2615 /* register unicast and multicast addresses */
2616 static void mvneta_set_rx_mode(struct net_device *dev)
2617 {
2618         struct mvneta_port *pp = netdev_priv(dev);
2619         struct netdev_hw_addr *ha;
2620 
2621         if (dev->flags & IFF_PROMISC) {
2622                 /* Accept all: Multicast + Unicast */
2623                 mvneta_rx_unicast_promisc_set(pp, 1);
2624                 mvneta_set_ucast_table(pp, pp->rxq_def);
2625                 mvneta_set_special_mcast_table(pp, pp->rxq_def);
2626                 mvneta_set_other_mcast_table(pp, pp->rxq_def);
2627         } else {
2628                 /* Accept single Unicast */
2629                 mvneta_rx_unicast_promisc_set(pp, 0);
2630                 mvneta_set_ucast_table(pp, -1);
2631                 mvneta_mac_addr_set(pp, dev->dev_addr, pp->rxq_def);
2632 
2633                 if (dev->flags & IFF_ALLMULTI) {
2634                         /* Accept all multicast */
2635                         mvneta_set_special_mcast_table(pp, pp->rxq_def);
2636                         mvneta_set_other_mcast_table(pp, pp->rxq_def);
2637                 } else {
2638                         /* Accept only initialized multicast */
2639                         mvneta_set_special_mcast_table(pp, -1);
2640                         mvneta_set_other_mcast_table(pp, -1);
2641 
2642                         if (!netdev_mc_empty(dev)) {
2643                                 netdev_for_each_mc_addr(ha, dev) {
2644                                         mvneta_mcast_addr_set(pp, ha->addr,
2645                                                               pp->rxq_def);
2646                                 }
2647                         }
2648                 }
2649         }
2650 }
2651 
2652 /* Interrupt handling - the callback for request_irq() */
2653 static irqreturn_t mvneta_isr(int irq, void *dev_id)
2654 {
2655         struct mvneta_port *pp = (struct mvneta_port *)dev_id;
2656 
2657         mvreg_write(pp, MVNETA_INTR_NEW_MASK, 0);
2658         napi_schedule(&pp->napi);
2659 
2660         return IRQ_HANDLED;
2661 }
2662 
2663 /* Interrupt handling - the callback for request_percpu_irq() */
2664 static irqreturn_t mvneta_percpu_isr(int irq, void *dev_id)
2665 {
2666         struct mvneta_pcpu_port *port = (struct mvneta_pcpu_port *)dev_id;
2667 
2668         disable_percpu_irq(port->pp->dev->irq);
2669         napi_schedule(&port->napi);
2670 
2671         return IRQ_HANDLED;
2672 }
2673 
2674 static int mvneta_fixed_link_update(struct mvneta_port *pp,
2675                                     struct phy_device *phy)
2676 {
2677         struct fixed_phy_status status;
2678         struct fixed_phy_status changed = {};
2679         u32 gmac_stat = mvreg_read(pp, MVNETA_GMAC_STATUS);
2680 
2681         status.link = !!(gmac_stat & MVNETA_GMAC_LINK_UP);
2682         if (gmac_stat & MVNETA_GMAC_SPEED_1000)
2683                 status.speed = SPEED_1000;
2684         else if (gmac_stat & MVNETA_GMAC_SPEED_100)
2685                 status.speed = SPEED_100;
2686         else
2687                 status.speed = SPEED_10;
2688         status.duplex = !!(gmac_stat & MVNETA_GMAC_FULL_DUPLEX);
2689         changed.link = 1;
2690         changed.speed = 1;
2691         changed.duplex = 1;
2692         fixed_phy_update_state(phy, &status, &changed);
2693         return 0;
2694 }
2695 
2696 /* NAPI handler
2697  * Bits 0 - 7 of the causeRxTx register indicate that are transmitted
2698  * packets on the corresponding TXQ (Bit 0 is for TX queue 1).
2699  * Bits 8 -15 of the cause Rx Tx register indicate that are received
2700  * packets on the corresponding RXQ (Bit 8 is for RX queue 0).
2701  * Each CPU has its own causeRxTx register
2702  */
2703 static int mvneta_poll(struct napi_struct *napi, int budget)
2704 {
2705         int rx_done = 0;
2706         u32 cause_rx_tx;
2707         int rx_queue;
2708         struct mvneta_port *pp = netdev_priv(napi->dev);
2709         struct net_device *ndev = pp->dev;
2710         struct mvneta_pcpu_port *port = this_cpu_ptr(pp->ports);
2711 
2712         if (!netif_running(pp->dev)) {
2713                 napi_complete(napi);
2714                 return rx_done;
2715         }
2716 
2717         /* Read cause register */
2718         cause_rx_tx = mvreg_read(pp, MVNETA_INTR_NEW_CAUSE);
2719         if (cause_rx_tx & MVNETA_MISCINTR_INTR_MASK) {
2720                 u32 cause_misc = mvreg_read(pp, MVNETA_INTR_MISC_CAUSE);
2721 
2722                 mvreg_write(pp, MVNETA_INTR_MISC_CAUSE, 0);
2723                 if (pp->use_inband_status && (cause_misc &
2724                                 (MVNETA_CAUSE_PHY_STATUS_CHANGE |
2725                                  MVNETA_CAUSE_LINK_CHANGE |
2726                                  MVNETA_CAUSE_PSC_SYNC_CHANGE))) {
2727                         mvneta_fixed_link_update(pp, ndev->phydev);
2728                 }
2729         }
2730 
2731         /* Release Tx descriptors */
2732         if (cause_rx_tx & MVNETA_TX_INTR_MASK_ALL) {
2733                 mvneta_tx_done_gbe(pp, (cause_rx_tx & MVNETA_TX_INTR_MASK_ALL));
2734                 cause_rx_tx &= ~MVNETA_TX_INTR_MASK_ALL;
2735         }
2736 
2737         /* For the case where the last mvneta_poll did not process all
2738          * RX packets
2739          */
2740         rx_queue = fls(((cause_rx_tx >> 8) & 0xff));
2741 
2742         cause_rx_tx |= pp->neta_armada3700 ? pp->cause_rx_tx :
2743                 port->cause_rx_tx;
2744 
2745         if (rx_queue) {
2746                 rx_queue = rx_queue - 1;
2747                 if (pp->bm_priv)
2748                         rx_done = mvneta_rx_hwbm(pp, budget, &pp->rxqs[rx_queue]);
2749                 else
2750                         rx_done = mvneta_rx_swbm(pp, budget, &pp->rxqs[rx_queue]);
2751         }
2752 
2753         budget -= rx_done;
2754 
2755         if (budget > 0) {
2756                 cause_rx_tx = 0;
2757                 napi_complete(napi);
2758 
2759                 if (pp->neta_armada3700) {
2760                         unsigned long flags;
2761 
2762                         local_irq_save(flags);
2763                         mvreg_write(pp, MVNETA_INTR_NEW_MASK,
2764                                     MVNETA_RX_INTR_MASK(rxq_number) |
2765                                     MVNETA_TX_INTR_MASK(txq_number) |
2766                                     MVNETA_MISCINTR_INTR_MASK);
2767                         local_irq_restore(flags);
2768                 } else {
2769                         enable_percpu_irq(pp->dev->irq, 0);
2770                 }
2771         }
2772 
2773         if (pp->neta_armada3700)
2774                 pp->cause_rx_tx = cause_rx_tx;
2775         else
2776                 port->cause_rx_tx = cause_rx_tx;
2777 
2778         return rx_done;
2779 }
2780 
2781 /* Handle rxq fill: allocates rxq skbs; called when initializing a port */
2782 static int mvneta_rxq_fill(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
2783                            int num)
2784 {
2785         int i;
2786 
2787         for (i = 0; i < num; i++) {
2788                 memset(rxq->descs + i, 0, sizeof(struct mvneta_rx_desc));
2789                 if (mvneta_rx_refill(pp, rxq->descs + i, rxq) != 0) {
2790                         netdev_err(pp->dev, "%s:rxq %d, %d of %d buffs  filled\n",
2791                                 __func__, rxq->id, i, num);
2792                         break;
2793                 }
2794         }
2795 
2796         /* Add this number of RX descriptors as non occupied (ready to
2797          * get packets)
2798          */
2799         mvneta_rxq_non_occup_desc_add(pp, rxq, i);
2800 
2801         return i;
2802 }
2803 
2804 /* Free all packets pending transmit from all TXQs and reset TX port */
2805 static void mvneta_tx_reset(struct mvneta_port *pp)
2806 {
2807         int queue;
2808 
2809         /* free the skb's in the tx ring */
2810         for (queue = 0; queue < txq_number; queue++)
2811                 mvneta_txq_done_force(pp, &pp->txqs[queue]);
2812 
2813         mvreg_write(pp, MVNETA_PORT_TX_RESET, MVNETA_PORT_TX_DMA_RESET);
2814         mvreg_write(pp, MVNETA_PORT_TX_RESET, 0);
2815 }
2816 
2817 static void mvneta_rx_reset(struct mvneta_port *pp)
2818 {
2819         mvreg_write(pp, MVNETA_PORT_RX_RESET, MVNETA_PORT_RX_DMA_RESET);
2820         mvreg_write(pp, MVNETA_PORT_RX_RESET, 0);
2821 }
2822 
2823 /* Rx/Tx queue initialization/cleanup methods */
2824 
2825 /* Create a specified RX queue */
2826 static int mvneta_rxq_init(struct mvneta_port *pp,
2827                            struct mvneta_rx_queue *rxq)
2828 
2829 {
2830         rxq->size = pp->rx_ring_size;
2831 
2832         /* Allocate memory for RX descriptors */
2833         rxq->descs = dma_alloc_coherent(pp->dev->dev.parent,
2834                                         rxq->size * MVNETA_DESC_ALIGNED_SIZE,
2835                                         &rxq->descs_phys, GFP_KERNEL);
2836         if (rxq->descs == NULL)
2837                 return -ENOMEM;
2838 
2839         rxq->last_desc = rxq->size - 1;
2840 
2841         /* Set Rx descriptors queue starting address */
2842         mvreg_write(pp, MVNETA_RXQ_BASE_ADDR_REG(rxq->id), rxq->descs_phys);
2843         mvreg_write(pp, MVNETA_RXQ_SIZE_REG(rxq->id), rxq->size);
2844 
2845         /* Set Offset */
2846         mvneta_rxq_offset_set(pp, rxq, NET_SKB_PAD - pp->rx_offset_correction);
2847 
2848         /* Set coalescing pkts and time */
2849         mvneta_rx_pkts_coal_set(pp, rxq, rxq->pkts_coal);
2850         mvneta_rx_time_coal_set(pp, rxq, rxq->time_coal);
2851 
2852         if (!pp->bm_priv) {
2853                 /* Fill RXQ with buffers from RX pool */
2854                 mvneta_rxq_buf_size_set(pp, rxq,
2855                                         MVNETA_RX_BUF_SIZE(pp->pkt_size));
2856                 mvneta_rxq_bm_disable(pp, rxq);
2857                 mvneta_rxq_fill(pp, rxq, rxq->size);
2858         } else {
2859                 mvneta_rxq_bm_enable(pp, rxq);
2860                 mvneta_rxq_long_pool_set(pp, rxq);
2861                 mvneta_rxq_short_pool_set(pp, rxq);
2862                 mvneta_rxq_non_occup_desc_add(pp, rxq, rxq->size);
2863         }
2864 
2865         return 0;
2866 }
2867 
2868 /* Cleanup Rx queue */
2869 static void mvneta_rxq_deinit(struct mvneta_port *pp,
2870                               struct mvneta_rx_queue *rxq)
2871 {
2872         mvneta_rxq_drop_pkts(pp, rxq);
2873 
2874         if (rxq->descs)
2875                 dma_free_coherent(pp->dev->dev.parent,
2876                                   rxq->size * MVNETA_DESC_ALIGNED_SIZE,
2877                                   rxq->descs,
2878                                   rxq->descs_phys);
2879 
2880         rxq->descs             = NULL;
2881         rxq->last_desc         = 0;
2882         rxq->next_desc_to_proc = 0;
2883         rxq->descs_phys        = 0;
2884 }
2885 
2886 /* Create and initialize a tx queue */
2887 static int mvneta_txq_init(struct mvneta_port *pp,
2888                            struct mvneta_tx_queue *txq)
2889 {
2890         int cpu;
2891 
2892         txq->size = pp->tx_ring_size;
2893 
2894         /* A queue must always have room for at least one skb.
2895          * Therefore, stop the queue when the free entries reaches
2896          * the maximum number of descriptors per skb.
2897          */
2898         txq->tx_stop_threshold = txq->size - MVNETA_MAX_SKB_DESCS;
2899         txq->tx_wake_threshold = txq->tx_stop_threshold / 2;
2900 
2901 
2902         /* Allocate memory for TX descriptors */
2903         txq->descs = dma_alloc_coherent(pp->dev->dev.parent,
2904                                         txq->size * MVNETA_DESC_ALIGNED_SIZE,
2905                                         &txq->descs_phys, GFP_KERNEL);
2906         if (txq->descs == NULL)
2907                 return -ENOMEM;
2908 
2909         txq->last_desc = txq->size - 1;
2910 
2911         /* Set maximum bandwidth for enabled TXQs */
2912         mvreg_write(pp, MVETH_TXQ_TOKEN_CFG_REG(txq->id), 0x03ffffff);
2913         mvreg_write(pp, MVETH_TXQ_TOKEN_COUNT_REG(txq->id), 0x3fffffff);
2914 
2915         /* Set Tx descriptors queue starting address */
2916         mvreg_write(pp, MVNETA_TXQ_BASE_ADDR_REG(txq->id), txq->descs_phys);
2917         mvreg_write(pp, MVNETA_TXQ_SIZE_REG(txq->id), txq->size);
2918 
2919         txq->tx_skb = kmalloc(txq->size * sizeof(*txq->tx_skb), GFP_KERNEL);
2920         if (txq->tx_skb == NULL) {
2921                 dma_free_coherent(pp->dev->dev.parent,
2922                                   txq->size * MVNETA_DESC_ALIGNED_SIZE,
2923                                   txq->descs, txq->descs_phys);
2924                 return -ENOMEM;
2925         }
2926 
2927         /* Allocate DMA buffers for TSO MAC/IP/TCP headers */
2928         txq->tso_hdrs = dma_alloc_coherent(pp->dev->dev.parent,
2929                                            txq->size * TSO_HEADER_SIZE,
2930                                            &txq->tso_hdrs_phys, GFP_KERNEL);
2931         if (txq->tso_hdrs == NULL) {
2932                 kfree(txq->tx_skb);
2933                 dma_free_coherent(pp->dev->dev.parent,
2934                                   txq->size * MVNETA_DESC_ALIGNED_SIZE,
2935                                   txq->descs, txq->descs_phys);
2936                 return -ENOMEM;
2937         }
2938         mvneta_tx_done_pkts_coal_set(pp, txq, txq->done_pkts_coal);
2939 
2940         /* Setup XPS mapping */
2941         if (txq_number > 1)
2942                 cpu = txq->id % num_present_cpus();
2943         else
2944                 cpu = pp->rxq_def % num_present_cpus();
2945         cpumask_set_cpu(cpu, &txq->affinity_mask);
2946         netif_set_xps_queue(pp->dev, &txq->affinity_mask, txq->id);
2947 
2948         return 0;
2949 }
2950 
2951 /* Free allocated resources when mvneta_txq_init() fails to allocate memory*/
2952 static void mvneta_txq_deinit(struct mvneta_port *pp,
2953                               struct mvneta_tx_queue *txq)
2954 {
2955         kfree(txq->tx_skb);
2956 
2957         if (txq->tso_hdrs)
2958                 dma_free_coherent(pp->dev->dev.parent,
2959                                   txq->size * TSO_HEADER_SIZE,
2960                                   txq->tso_hdrs, txq->tso_hdrs_phys);
2961         if (txq->descs)
2962                 dma_free_coherent(pp->dev->dev.parent,
2963                                   txq->size * MVNETA_DESC_ALIGNED_SIZE,
2964                                   txq->descs, txq->descs_phys);
2965 
2966         txq->descs             = NULL;
2967         txq->last_desc         = 0;
2968         txq->next_desc_to_proc = 0;
2969         txq->descs_phys        = 0;
2970 
2971         /* Set minimum bandwidth for disabled TXQs */
2972         mvreg_write(pp, MVETH_TXQ_TOKEN_CFG_REG(txq->id), 0);
2973         mvreg_write(pp, MVETH_TXQ_TOKEN_COUNT_REG(txq->id), 0);
2974 
2975         /* Set Tx descriptors queue starting address and size */
2976         mvreg_write(pp, MVNETA_TXQ_BASE_ADDR_REG(txq->id), 0);
2977         mvreg_write(pp, MVNETA_TXQ_SIZE_REG(txq->id), 0);
2978 }
2979 
2980 /* Cleanup all Tx queues */
2981 static void mvneta_cleanup_txqs(struct mvneta_port *pp)
2982 {
2983         int queue;
2984 
2985         for (queue = 0; queue < txq_number; queue++)
2986                 mvneta_txq_deinit(pp, &pp->txqs[queue]);
2987 }
2988 
2989 /* Cleanup all Rx queues */
2990 static void mvneta_cleanup_rxqs(struct mvneta_port *pp)
2991 {
2992         int queue;
2993 
2994         for (queue = 0; queue < txq_number; queue++)
2995                 mvneta_rxq_deinit(pp, &pp->rxqs[queue]);
2996 }
2997 
2998 
2999 /* Init all Rx queues */
3000 static int mvneta_setup_rxqs(struct mvneta_port *pp)
3001 {
3002         int queue;
3003 
3004         for (queue = 0; queue < rxq_number; queue++) {
3005                 int err = mvneta_rxq_init(pp, &pp->rxqs[queue]);
3006 
3007                 if (err) {
3008                         netdev_err(pp->dev, "%s: can't create rxq=%d\n",
3009                                    __func__, queue);
3010                         mvneta_cleanup_rxqs(pp);
3011                         return err;
3012                 }
3013         }
3014 
3015         return 0;
3016 }
3017 
3018 /* Init all tx queues */
3019 static int mvneta_setup_txqs(struct mvneta_port *pp)
3020 {
3021         int queue;
3022 
3023         for (queue = 0; queue < txq_number; queue++) {
3024                 int err = mvneta_txq_init(pp, &pp->txqs[queue]);
3025                 if (err) {
3026                         netdev_err(pp->dev, "%s: can't create txq=%d\n",
3027                                    __func__, queue);
3028                         mvneta_cleanup_txqs(pp);
3029                         return err;
3030                 }
3031         }
3032 
3033         return 0;
3034 }
3035 
3036 static void mvneta_start_dev(struct mvneta_port *pp)
3037 {
3038         int cpu;
3039         struct net_device *ndev = pp->dev;
3040 
3041         mvneta_max_rx_size_set(pp, pp->pkt_size);
3042         mvneta_txq_max_tx_size_set(pp, pp->pkt_size);
3043 
3044         /* start the Rx/Tx activity */
3045         mvneta_port_enable(pp);
3046 
3047         if (!pp->neta_armada3700) {
3048                 /* Enable polling on the port */
3049                 for_each_online_cpu(cpu) {
3050                         struct mvneta_pcpu_port *port =
3051                                 per_cpu_ptr(pp->ports, cpu);
3052 
3053                         napi_enable(&port->napi);
3054                 }
3055         } else {
3056                 napi_enable(&pp->napi);
3057         }
3058 
3059         /* Unmask interrupts. It has to be done from each CPU */
3060         on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true);
3061 
3062         mvreg_write(pp, MVNETA_INTR_MISC_MASK,
3063                     MVNETA_CAUSE_PHY_STATUS_CHANGE |
3064                     MVNETA_CAUSE_LINK_CHANGE |
3065                     MVNETA_CAUSE_PSC_SYNC_CHANGE);
3066 
3067         phy_start(ndev->phydev);
3068         netif_tx_start_all_queues(pp->dev);
3069 }
3070 
3071 static void mvneta_stop_dev(struct mvneta_port *pp)
3072 {
3073         unsigned int cpu;
3074         struct net_device *ndev = pp->dev;
3075 
3076         phy_stop(ndev->phydev);
3077 
3078         if (!pp->neta_armada3700) {
3079                 for_each_online_cpu(cpu) {
3080                         struct mvneta_pcpu_port *port =
3081                                 per_cpu_ptr(pp->ports, cpu);
3082 
3083                         napi_disable(&port->napi);
3084                 }
3085         } else {
3086                 napi_disable(&pp->napi);
3087         }
3088 
3089         netif_carrier_off(pp->dev);
3090 
3091         mvneta_port_down(pp);
3092         netif_tx_stop_all_queues(pp->dev);
3093 
3094         /* Stop the port activity */
3095         mvneta_port_disable(pp);
3096 
3097         /* Clear all ethernet port interrupts */
3098         on_each_cpu(mvneta_percpu_clear_intr_cause, pp, true);
3099 
3100         /* Mask all ethernet port interrupts */
3101         on_each_cpu(mvneta_percpu_mask_interrupt, pp, true);
3102 
3103         mvneta_tx_reset(pp);
3104         mvneta_rx_reset(pp);
3105 }
3106 
3107 static void mvneta_percpu_enable(void *arg)
3108 {
3109         struct mvneta_port *pp = arg;
3110 
3111         enable_percpu_irq(pp->dev->irq, IRQ_TYPE_NONE);
3112 }
3113 
3114 static void mvneta_percpu_disable(void *arg)
3115 {
3116         struct mvneta_port *pp = arg;
3117 
3118         disable_percpu_irq(pp->dev->irq);
3119 }
3120 
3121 /* Change the device mtu */
3122 static int mvneta_change_mtu(struct net_device *dev, int mtu)
3123 {
3124         struct mvneta_port *pp = netdev_priv(dev);
3125         int ret;
3126 
3127         if (!IS_ALIGNED(MVNETA_RX_PKT_SIZE(mtu), 8)) {
3128                 netdev_info(dev, "Illegal MTU value %d, rounding to %d\n",
3129                             mtu, ALIGN(MVNETA_RX_PKT_SIZE(mtu), 8));
3130                 mtu = ALIGN(MVNETA_RX_PKT_SIZE(mtu), 8);
3131         }
3132 
3133         dev->mtu = mtu;
3134 
3135         if (!netif_running(dev)) {
3136                 if (pp->bm_priv)
3137                         mvneta_bm_update_mtu(pp, mtu);
3138 
3139                 netdev_update_features(dev);
3140                 return 0;
3141         }
3142 
3143         /* The interface is running, so we have to force a
3144          * reallocation of the queues
3145          */
3146         mvneta_stop_dev(pp);
3147         on_each_cpu(mvneta_percpu_disable, pp, true);
3148 
3149         mvneta_cleanup_txqs(pp);
3150         mvneta_cleanup_rxqs(pp);
3151 
3152         if (pp->bm_priv)
3153                 mvneta_bm_update_mtu(pp, mtu);
3154 
3155         pp->pkt_size = MVNETA_RX_PKT_SIZE(dev->mtu);
3156         pp->frag_size = SKB_DATA_ALIGN(MVNETA_RX_BUF_SIZE(pp->pkt_size)) +
3157                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3158 
3159         ret = mvneta_setup_rxqs(pp);
3160         if (ret) {
3161                 netdev_err(dev, "unable to setup rxqs after MTU change\n");
3162                 return ret;
3163         }
3164 
3165         ret = mvneta_setup_txqs(pp);
3166         if (ret) {
3167                 netdev_err(dev, "unable to setup txqs after MTU change\n");
3168                 return ret;
3169         }
3170 
3171         on_each_cpu(mvneta_percpu_enable, pp, true);
3172         mvneta_start_dev(pp);
3173         mvneta_port_up(pp);
3174 
3175         netdev_update_features(dev);
3176 
3177         return 0;
3178 }
3179 
3180 static netdev_features_t mvneta_fix_features(struct net_device *dev,
3181                                              netdev_features_t features)
3182 {
3183         struct mvneta_port *pp = netdev_priv(dev);
3184 
3185         if (pp->tx_csum_limit && dev->mtu > pp->tx_csum_limit) {
3186                 features &= ~(NETIF_F_IP_CSUM | NETIF_F_TSO);
3187                 netdev_info(dev,
3188                             "Disable IP checksum for MTU greater than %dB\n",
3189                             pp->tx_csum_limit);
3190         }
3191 
3192         return features;
3193 }
3194 
3195 /* Get mac address */
3196 static void mvneta_get_mac_addr(struct mvneta_port *pp, unsigned char *addr)
3197 {
3198         u32 mac_addr_l, mac_addr_h;
3199 
3200         mac_addr_l = mvreg_read(pp, MVNETA_MAC_ADDR_LOW);
3201         mac_addr_h = mvreg_read(pp, MVNETA_MAC_ADDR_HIGH);
3202         addr[0] = (mac_addr_h >> 24) & 0xFF;
3203         addr[1] = (mac_addr_h >> 16) & 0xFF;
3204         addr[2] = (mac_addr_h >> 8) & 0xFF;
3205         addr[3] = mac_addr_h & 0xFF;
3206         addr[4] = (mac_addr_l >> 8) & 0xFF;
3207         addr[5] = mac_addr_l & 0xFF;
3208 }
3209 
3210 /* Handle setting mac address */
3211 static int mvneta_set_mac_addr(struct net_device *dev, void *addr)
3212 {
3213         struct mvneta_port *pp = netdev_priv(dev);
3214         struct sockaddr *sockaddr = addr;
3215         int ret;
3216 
3217         ret = eth_prepare_mac_addr_change(dev, addr);
3218         if (ret < 0)
3219                 return ret;
3220         /* Remove previous address table entry */
3221         mvneta_mac_addr_set(pp, dev->dev_addr, -1);
3222 
3223         /* Set new addr in hw */
3224         mvneta_mac_addr_set(pp, sockaddr->sa_data, pp->rxq_def);
3225 
3226         eth_commit_mac_addr_change(dev, addr);
3227         return 0;
3228 }
3229 
3230 static void mvneta_adjust_link(struct net_device *ndev)
3231 {
3232         struct mvneta_port *pp = netdev_priv(ndev);
3233         struct phy_device *phydev = ndev->phydev;
3234         int status_change = 0;
3235 
3236         if (phydev->link) {
3237                 if ((pp->speed != phydev->speed) ||
3238                     (pp->duplex != phydev->duplex)) {
3239                         u32 val;
3240 
3241                         val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
3242                         val &= ~(MVNETA_GMAC_CONFIG_MII_SPEED |
3243                                  MVNETA_GMAC_CONFIG_GMII_SPEED |
3244                                  MVNETA_GMAC_CONFIG_FULL_DUPLEX);
3245 
3246                         if (phydev->duplex)
3247                                 val |= MVNETA_GMAC_CONFIG_FULL_DUPLEX;
3248 
3249                         if (phydev->speed == SPEED_1000)
3250                                 val |= MVNETA_GMAC_CONFIG_GMII_SPEED;
3251                         else if (phydev->speed == SPEED_100)
3252                                 val |= MVNETA_GMAC_CONFIG_MII_SPEED;
3253 
3254                         mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
3255 
3256                         pp->duplex = phydev->duplex;
3257                         pp->speed  = phydev->speed;
3258                 }
3259         }
3260 
3261         if (phydev->link != pp->link) {
3262                 if (!phydev->link) {
3263                         pp->duplex = -1;
3264                         pp->speed = 0;
3265                 }
3266 
3267                 pp->link = phydev->link;
3268                 status_change = 1;
3269         }
3270 
3271         if (status_change) {
3272                 if (phydev->link) {
3273                         if (!pp->use_inband_status) {
3274                                 u32 val = mvreg_read(pp,
3275                                                   MVNETA_GMAC_AUTONEG_CONFIG);
3276                                 val &= ~MVNETA_GMAC_FORCE_LINK_DOWN;
3277                                 val |= MVNETA_GMAC_FORCE_LINK_PASS;
3278                                 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG,
3279                                             val);
3280                         }
3281                         mvneta_port_up(pp);
3282                 } else {
3283                         if (!pp->use_inband_status) {
3284                                 u32 val = mvreg_read(pp,
3285                                                   MVNETA_GMAC_AUTONEG_CONFIG);
3286                                 val &= ~MVNETA_GMAC_FORCE_LINK_PASS;
3287                                 val |= MVNETA_GMAC_FORCE_LINK_DOWN;
3288                                 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG,
3289                                             val);
3290                         }
3291                         mvneta_port_down(pp);
3292                 }
3293                 phy_print_status(phydev);
3294         }
3295 }
3296 
3297 static int mvneta_mdio_probe(struct mvneta_port *pp)
3298 {
3299         struct phy_device *phy_dev;
3300 
3301         phy_dev = of_phy_connect(pp->dev, pp->phy_node, mvneta_adjust_link, 0,
3302                                  pp->phy_interface);
3303         if (!phy_dev) {
3304                 netdev_err(pp->dev, "could not find the PHY\n");
3305                 return -ENODEV;
3306         }
3307 
3308         phy_dev->supported &= PHY_GBIT_FEATURES;
3309         phy_dev->advertising = phy_dev->supported;
3310 
3311         pp->link    = 0;
3312         pp->duplex  = 0;
3313         pp->speed   = 0;
3314 
3315         return 0;
3316 }
3317 
3318 static void mvneta_mdio_remove(struct mvneta_port *pp)
3319 {
3320         struct net_device *ndev = pp->dev;
3321 
3322         phy_disconnect(ndev->phydev);
3323 }
3324 
3325 /* Electing a CPU must be done in an atomic way: it should be done
3326  * after or before the removal/insertion of a CPU and this function is
3327  * not reentrant.
3328  */
3329 static void mvneta_percpu_elect(struct mvneta_port *pp)
3330 {
3331         int elected_cpu = 0, max_cpu, cpu, i = 0;
3332 
3333         /* Use the cpu associated to the rxq when it is online, in all
3334          * the other cases, use the cpu 0 which can't be offline.
3335          */
3336         if (cpu_online(pp->rxq_def))
3337                 elected_cpu = pp->rxq_def;
3338 
3339         max_cpu = num_present_cpus();
3340 
3341         for_each_online_cpu(cpu) {
3342                 int rxq_map = 0, txq_map = 0;
3343                 int rxq;
3344 
3345                 for (rxq = 0; rxq < rxq_number; rxq++)
3346                         if ((rxq % max_cpu) == cpu)
3347                                 rxq_map |= MVNETA_CPU_RXQ_ACCESS(rxq);
3348 
3349                 if (cpu == elected_cpu)
3350                         /* Map the default receive queue queue to the
3351                          * elected CPU
3352                          */
3353                         rxq_map |= MVNETA_CPU_RXQ_ACCESS(pp->rxq_def);
3354 
3355                 /* We update the TX queue map only if we have one
3356                  * queue. In this case we associate the TX queue to
3357                  * the CPU bound to the default RX queue
3358                  */
3359                 if (txq_number == 1)
3360                         txq_map = (cpu == elected_cpu) ?
3361                                 MVNETA_CPU_TXQ_ACCESS(1) : 0;
3362                 else
3363                         txq_map = mvreg_read(pp, MVNETA_CPU_MAP(cpu)) &
3364                                 MVNETA_CPU_TXQ_ACCESS_ALL_MASK;
3365 
3366                 mvreg_write(pp, MVNETA_CPU_MAP(cpu), rxq_map | txq_map);
3367 
3368                 /* Update the interrupt mask on each CPU according the
3369                  * new mapping
3370                  */
3371                 smp_call_function_single(cpu, mvneta_percpu_unmask_interrupt,
3372                                          pp, true);
3373                 i++;
3374 
3375         }
3376 };
3377 
3378 static int mvneta_cpu_online(unsigned int cpu, struct hlist_node *node)
3379 {
3380         int other_cpu;
3381         struct mvneta_port *pp = hlist_entry_safe(node, struct mvneta_port,
3382                                                   node_online);
3383         struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu);
3384 
3385 
3386         spin_lock(&pp->lock);
3387         /*
3388          * Configuring the driver for a new CPU while the driver is
3389          * stopping is racy, so just avoid it.
3390          */
3391         if (pp->is_stopped) {
3392                 spin_unlock(&pp->lock);
3393                 return 0;
3394         }
3395         netif_tx_stop_all_queues(pp->dev);
3396 
3397         /*
3398          * We have to synchronise on tha napi of each CPU except the one
3399          * just being woken up
3400          */
3401         for_each_online_cpu(other_cpu) {
3402                 if (other_cpu != cpu) {
3403                         struct mvneta_pcpu_port *other_port =
3404                                 per_cpu_ptr(pp->ports, other_cpu);
3405 
3406                         napi_synchronize(&other_port->napi);
3407                 }
3408         }
3409 
3410         /* Mask all ethernet port interrupts */
3411         on_each_cpu(mvneta_percpu_mask_interrupt, pp, true);
3412         napi_enable(&port->napi);
3413 
3414         /*
3415          * Enable per-CPU interrupts on the CPU that is
3416          * brought up.
3417          */
3418         mvneta_percpu_enable(pp);
3419 
3420         /*
3421          * Enable per-CPU interrupt on the one CPU we care
3422          * about.
3423          */
3424         mvneta_percpu_elect(pp);
3425 
3426         /* Unmask all ethernet port interrupts */
3427         on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true);
3428         mvreg_write(pp, MVNETA_INTR_MISC_MASK,
3429                     MVNETA_CAUSE_PHY_STATUS_CHANGE |
3430                     MVNETA_CAUSE_LINK_CHANGE |
3431                     MVNETA_CAUSE_PSC_SYNC_CHANGE);
3432         netif_tx_start_all_queues(pp->dev);
3433         spin_unlock(&pp->lock);
3434         return 0;
3435 }
3436 
3437 static int mvneta_cpu_down_prepare(unsigned int cpu, struct hlist_node *node)
3438 {
3439         struct mvneta_port *pp = hlist_entry_safe(node, struct mvneta_port,
3440                                                   node_online);
3441         struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu);
3442 
3443         /*
3444          * Thanks to this lock we are sure that any pending cpu election is
3445          * done.
3446          */
3447         spin_lock(&pp->lock);
3448         /* Mask all ethernet port interrupts */
3449         on_each_cpu(mvneta_percpu_mask_interrupt, pp, true);
3450         spin_unlock(&pp->lock);
3451 
3452         napi_synchronize(&port->napi);
3453         napi_disable(&port->napi);
3454         /* Disable per-CPU interrupts on the CPU that is brought down. */
3455         mvneta_percpu_disable(pp);
3456         return 0;
3457 }
3458 
3459 static int mvneta_cpu_dead(unsigned int cpu, struct hlist_node *node)
3460 {
3461         struct mvneta_port *pp = hlist_entry_safe(node, struct mvneta_port,
3462                                                   node_dead);
3463 
3464         /* Check if a new CPU must be elected now this on is down */
3465         spin_lock(&pp->lock);
3466         mvneta_percpu_elect(pp);
3467         spin_unlock(&pp->lock);
3468         /* Unmask all ethernet port interrupts */
3469         on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true);
3470         mvreg_write(pp, MVNETA_INTR_MISC_MASK,
3471                     MVNETA_CAUSE_PHY_STATUS_CHANGE |
3472                     MVNETA_CAUSE_LINK_CHANGE |
3473                     MVNETA_CAUSE_PSC_SYNC_CHANGE);
3474         netif_tx_start_all_queues(pp->dev);
3475         return 0;
3476 }
3477 
3478 static int mvneta_open(struct net_device *dev)
3479 {
3480         struct mvneta_port *pp = netdev_priv(dev);
3481         int ret;
3482 
3483         pp->pkt_size = MVNETA_RX_PKT_SIZE(pp->dev->mtu);
3484         pp->frag_size = SKB_DATA_ALIGN(MVNETA_RX_BUF_SIZE(pp->pkt_size)) +
3485                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3486 
3487         ret = mvneta_setup_rxqs(pp);
3488         if (ret)
3489                 return ret;
3490 
3491         ret = mvneta_setup_txqs(pp);
3492         if (ret)
3493                 goto err_cleanup_rxqs;
3494 
3495         /* Connect to port interrupt line */
3496         if (pp->neta_armada3700)
3497                 ret = request_irq(pp->dev->irq, mvneta_isr, 0,
3498                                   dev->name, pp);
3499         else
3500                 ret = request_percpu_irq(pp->dev->irq, mvneta_percpu_isr,
3501                                          dev->name, pp->ports);
3502         if (ret) {
3503                 netdev_err(pp->dev, "cannot request irq %d\n", pp->dev->irq);
3504                 goto err_cleanup_txqs;
3505         }
3506 
3507         if (!pp->neta_armada3700) {
3508                 /* Enable per-CPU interrupt on all the CPU to handle our RX
3509                  * queue interrupts
3510                  */
3511                 on_each_cpu(mvneta_percpu_enable, pp, true);
3512 
3513                 pp->is_stopped = false;
3514                 /* Register a CPU notifier to handle the case where our CPU
3515                  * might be taken offline.
3516                  */
3517                 ret = cpuhp_state_add_instance_nocalls(online_hpstate,
3518                                                        &pp->node_online);
3519                 if (ret)
3520                         goto err_free_irq;
3521 
3522                 ret = cpuhp_state_add_instance_nocalls(CPUHP_NET_MVNETA_DEAD,
3523                                                        &pp->node_dead);
3524                 if (ret)
3525                         goto err_free_online_hp;
3526         }
3527 
3528         /* In default link is down */
3529         netif_carrier_off(pp->dev);
3530 
3531         ret = mvneta_mdio_probe(pp);
3532         if (ret < 0) {
3533                 netdev_err(dev, "cannot probe MDIO bus\n");
3534                 goto err_free_dead_hp;
3535         }
3536 
3537         mvneta_start_dev(pp);
3538 
3539         return 0;
3540 
3541 err_free_dead_hp:
3542         if (!pp->neta_armada3700)
3543                 cpuhp_state_remove_instance_nocalls(CPUHP_NET_MVNETA_DEAD,
3544                                                     &pp->node_dead);
3545 err_free_online_hp:
3546         if (!pp->neta_armada3700)
3547                 cpuhp_state_remove_instance_nocalls(online_hpstate,
3548                                                     &pp->node_online);
3549 err_free_irq:
3550         if (pp->neta_armada3700) {
3551                 free_irq(pp->dev->irq, pp);
3552         } else {
3553                 on_each_cpu(mvneta_percpu_disable, pp, true);
3554                 free_percpu_irq(pp->dev->irq, pp->ports);
3555         }
3556 err_cleanup_txqs:
3557         mvneta_cleanup_txqs(pp);
3558 err_cleanup_rxqs:
3559         mvneta_cleanup_rxqs(pp);
3560         return ret;
3561 }
3562 
3563 /* Stop the port, free port interrupt line */
3564 static int mvneta_stop(struct net_device *dev)
3565 {
3566         struct mvneta_port *pp = netdev_priv(dev);
3567 
3568         if (!pp->neta_armada3700) {
3569                 /* Inform that we are stopping so we don't want to setup the
3570                  * driver for new CPUs in the notifiers. The code of the
3571                  * notifier for CPU online is protected by the same spinlock,
3572                  * so when we get the lock, the notifer work is done.
3573                  */
3574                 spin_lock(&pp->lock);
3575                 pp->is_stopped = true;
3576                 spin_unlock(&pp->lock);
3577 
3578                 mvneta_stop_dev(pp);
3579                 mvneta_mdio_remove(pp);
3580 
3581                 cpuhp_state_remove_instance_nocalls(online_hpstate,
3582                                                     &pp->node_online);
3583                 cpuhp_state_remove_instance_nocalls(CPUHP_NET_MVNETA_DEAD,
3584                                                     &pp->node_dead);
3585                 on_each_cpu(mvneta_percpu_disable, pp, true);
3586                 free_percpu_irq(dev->irq, pp->ports);
3587         } else {
3588                 mvneta_stop_dev(pp);
3589                 mvneta_mdio_remove(pp);
3590                 free_irq(dev->irq, pp);
3591         }
3592 
3593         mvneta_cleanup_rxqs(pp);
3594         mvneta_cleanup_txqs(pp);
3595 
3596         return 0;
3597 }
3598 
3599 static int mvneta_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
3600 {
3601         if (!dev->phydev)
3602                 return -ENOTSUPP;
3603 
3604         return phy_mii_ioctl(dev->phydev, ifr, cmd);
3605 }
3606 
3607 /* Ethtool methods */
3608 
3609 /* Set link ksettings (phy address, speed) for ethtools */
3610 static int
3611 mvneta_ethtool_set_link_ksettings(struct net_device *ndev,
3612                                   const struct ethtool_link_ksettings *cmd)
3613 {
3614         struct mvneta_port *pp = netdev_priv(ndev);
3615         struct phy_device *phydev = ndev->phydev;
3616 
3617         if (!phydev)
3618                 return -ENODEV;
3619 
3620         if ((cmd->base.autoneg == AUTONEG_ENABLE) != pp->use_inband_status) {
3621                 u32 val;
3622 
3623                 mvneta_set_autoneg(pp, cmd->base.autoneg == AUTONEG_ENABLE);
3624 
3625                 if (cmd->base.autoneg == AUTONEG_DISABLE) {
3626                         val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
3627                         val &= ~(MVNETA_GMAC_CONFIG_MII_SPEED |
3628                                  MVNETA_GMAC_CONFIG_GMII_SPEED |
3629                                  MVNETA_GMAC_CONFIG_FULL_DUPLEX);
3630 
3631                         if (phydev->duplex)
3632                                 val |= MVNETA_GMAC_CONFIG_FULL_DUPLEX;
3633 
3634                         if (phydev->speed == SPEED_1000)
3635                                 val |= MVNETA_GMAC_CONFIG_GMII_SPEED;
3636                         else if (phydev->speed == SPEED_100)
3637                                 val |= MVNETA_GMAC_CONFIG_MII_SPEED;
3638 
3639                         mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
3640                 }
3641 
3642                 pp->use_inband_status = (cmd->base.autoneg == AUTONEG_ENABLE);
3643                 netdev_info(pp->dev, "autoneg status set to %i\n",
3644                             pp->use_inband_status);
3645 
3646                 if (netif_running(ndev)) {
3647                         mvneta_port_down(pp);
3648                         mvneta_port_up(pp);
3649                 }
3650         }
3651 
3652         return phy_ethtool_ksettings_set(ndev->phydev, cmd);
3653 }
3654 
3655 /* Set interrupt coalescing for ethtools */
3656 static int mvneta_ethtool_set_coalesce(struct net_device *dev,
3657                                        struct ethtool_coalesce *c)
3658 {
3659         struct mvneta_port *pp = netdev_priv(dev);
3660         int queue;
3661 
3662         for (queue = 0; queue < rxq_number; queue++) {
3663                 struct mvneta_rx_queue *rxq = &pp->rxqs[queue];
3664                 rxq->time_coal = c->rx_coalesce_usecs;
3665                 rxq->pkts_coal = c->rx_max_coalesced_frames;
3666                 mvneta_rx_pkts_coal_set(pp, rxq, rxq->pkts_coal);
3667                 mvneta_rx_time_coal_set(pp, rxq, rxq->time_coal);
3668         }
3669 
3670         for (queue = 0; queue < txq_number; queue++) {
3671                 struct mvneta_tx_queue *txq = &pp->txqs[queue];
3672                 txq->done_pkts_coal = c->tx_max_coalesced_frames;
3673                 mvneta_tx_done_pkts_coal_set(pp, txq, txq->done_pkts_coal);
3674         }
3675 
3676         return 0;
3677 }
3678 
3679 /* get coalescing for ethtools */
3680 static int mvneta_ethtool_get_coalesce(struct net_device *dev,
3681                                        struct ethtool_coalesce *c)
3682 {
3683         struct mvneta_port *pp = netdev_priv(dev);
3684 
3685         c->rx_coalesce_usecs        = pp->rxqs[0].time_coal;
3686         c->rx_max_coalesced_frames  = pp->rxqs[0].pkts_coal;
3687 
3688         c->tx_max_coalesced_frames =  pp->txqs[0].done_pkts_coal;
3689         return 0;
3690 }
3691 
3692 
3693 static void mvneta_ethtool_get_drvinfo(struct net_device *dev,
3694                                     struct ethtool_drvinfo *drvinfo)
3695 {
3696         strlcpy(drvinfo->driver, MVNETA_DRIVER_NAME,
3697                 sizeof(drvinfo->driver));
3698         strlcpy(drvinfo->version, MVNETA_DRIVER_VERSION,
3699                 sizeof(drvinfo->version));
3700         strlcpy(drvinfo->bus_info, dev_name(&dev->dev),
3701                 sizeof(drvinfo->bus_info));
3702 }
3703 
3704 
3705 static void mvneta_ethtool_get_ringparam(struct net_device *netdev,
3706                                          struct ethtool_ringparam *ring)
3707 {
3708         struct mvneta_port *pp = netdev_priv(netdev);
3709 
3710         ring->rx_max_pending = MVNETA_MAX_RXD;
3711         ring->tx_max_pending = MVNETA_MAX_TXD;
3712         ring->rx_pending = pp->rx_ring_size;
3713         ring->tx_pending = pp->tx_ring_size;
3714 }
3715 
3716 static int mvneta_ethtool_set_ringparam(struct net_device *dev,
3717                                         struct ethtool_ringparam *ring)
3718 {
3719         struct mvneta_port *pp = netdev_priv(dev);
3720 
3721         if ((ring->rx_pending == 0) || (ring->tx_pending == 0))
3722                 return -EINVAL;
3723         pp->rx_ring_size = ring->rx_pending < MVNETA_MAX_RXD ?
3724                 ring->rx_pending : MVNETA_MAX_RXD;
3725 
3726         pp->tx_ring_size = clamp_t(u16, ring->tx_pending,
3727                                    MVNETA_MAX_SKB_DESCS * 2, MVNETA_MAX_TXD);
3728         if (pp->tx_ring_size != ring->tx_pending)
3729                 netdev_warn(dev, "TX queue size set to %u (requested %u)\n",
3730                             pp->tx_ring_size, ring->tx_pending);
3731 
3732         if (netif_running(dev)) {
3733                 mvneta_stop(dev);
3734                 if (mvneta_open(dev)) {
3735                         netdev_err(dev,
3736                                    "error on opening device after ring param change\n");
3737                         return -ENOMEM;
3738                 }
3739         }
3740 
3741         return 0;
3742 }
3743 
3744 static void mvneta_ethtool_get_strings(struct net_device *netdev, u32 sset,
3745                                        u8 *data)
3746 {
3747         if (sset == ETH_SS_STATS) {
3748                 int i;
3749 
3750                 for (i = 0; i < ARRAY_SIZE(mvneta_statistics); i++)
3751                         memcpy(data + i * ETH_GSTRING_LEN,
3752                                mvneta_statistics[i].name, ETH_GSTRING_LEN);
3753         }
3754 }
3755 
3756 static void mvneta_ethtool_update_stats(struct mvneta_port *pp)
3757 {
3758         const struct mvneta_statistic *s;
3759         void __iomem *base = pp->base;
3760         u32 high, low, val;
3761         u64 val64;
3762         int i;
3763 
3764         for (i = 0, s = mvneta_statistics;
3765              s < mvneta_statistics + ARRAY_SIZE(mvneta_statistics);
3766              s++, i++) {
3767                 switch (s->type) {
3768                 case T_REG_32:
3769                         val = readl_relaxed(base + s->offset);
3770                         pp->ethtool_stats[i] += val;
3771                         break;
3772                 case T_REG_64:
3773                         /* Docs say to read low 32-bit then high */
3774                         low = readl_relaxed(base + s->offset);
3775                         high = readl_relaxed(base + s->offset + 4);
3776                         val64 = (u64)high << 32 | low;
3777                         pp->ethtool_stats[i] += val64;
3778                         break;
3779                 }
3780         }
3781 }
3782 
3783 static void mvneta_ethtool_get_stats(struct net_device *dev,
3784                                      struct ethtool_stats *stats, u64 *data)
3785 {
3786         struct mvneta_port *pp = netdev_priv(dev);
3787         int i;
3788 
3789         mvneta_ethtool_update_stats(pp);
3790 
3791         for (i = 0; i < ARRAY_SIZE(mvneta_statistics); i++)
3792                 *data++ = pp->ethtool_stats[i];
3793 }
3794 
3795 static int mvneta_ethtool_get_sset_count(struct net_device *dev, int sset)
3796 {
3797         if (sset == ETH_SS_STATS)
3798                 return ARRAY_SIZE(mvneta_statistics);
3799         return -EOPNOTSUPP;
3800 }
3801 
3802 static u32 mvneta_ethtool_get_rxfh_indir_size(struct net_device *dev)
3803 {
3804         return MVNETA_RSS_LU_TABLE_SIZE;
3805 }
3806 
3807 static int mvneta_ethtool_get_rxnfc(struct net_device *dev,
3808                                     struct ethtool_rxnfc *info,
3809                                     u32 *rules __always_unused)
3810 {
3811         switch (info->cmd) {
3812         case ETHTOOL_GRXRINGS:
3813                 info->data =  rxq_number;
3814                 return 0;
3815         case ETHTOOL_GRXFH:
3816                 return -EOPNOTSUPP;
3817         default:
3818                 return -EOPNOTSUPP;
3819         }
3820 }
3821 
3822 static int  mvneta_config_rss(struct mvneta_port *pp)
3823 {
3824         int cpu;
3825         u32 val;
3826 
3827         netif_tx_stop_all_queues(pp->dev);
3828 
3829         on_each_cpu(mvneta_percpu_mask_interrupt, pp, true);
3830 
3831         /* We have to synchronise on the napi of each CPU */
3832         for_each_online_cpu(cpu) {
3833                 struct mvneta_pcpu_port *pcpu_port =
3834                         per_cpu_ptr(pp->ports, cpu);
3835 
3836                 napi_synchronize(&pcpu_port->napi);
3837                 napi_disable(&pcpu_port->napi);
3838         }
3839 
3840         pp->rxq_def = pp->indir[0];
3841 
3842         /* Update unicast mapping */
3843         mvneta_set_rx_mode(pp->dev);
3844 
3845         /* Update val of portCfg register accordingly with all RxQueue types */
3846         val = MVNETA_PORT_CONFIG_DEFL_VALUE(pp->rxq_def);
3847         mvreg_write(pp, MVNETA_PORT_CONFIG, val);
3848 
3849         /* Update the elected CPU matching the new rxq_def */
3850         spin_lock(&pp->lock);
3851         mvneta_percpu_elect(pp);
3852         spin_unlock(&pp->lock);
3853 
3854         /* We have to synchronise on the napi of each CPU */
3855         for_each_online_cpu(cpu) {
3856                 struct mvneta_pcpu_port *pcpu_port =
3857                         per_cpu_ptr(pp->ports, cpu);
3858 
3859                 napi_enable(&pcpu_port->napi);
3860         }
3861 
3862         netif_tx_start_all_queues(pp->dev);
3863 
3864         return 0;
3865 }
3866 
3867 static int mvneta_ethtool_set_rxfh(struct net_device *dev, const u32 *indir,
3868                                    const u8 *key, const u8 hfunc)
3869 {
3870         struct mvneta_port *pp = netdev_priv(dev);
3871 
3872         /* Current code for Armada 3700 doesn't support RSS features yet */
3873         if (pp->neta_armada3700)
3874                 return -EOPNOTSUPP;
3875 
3876         /* We require at least one supported parameter to be changed
3877          * and no change in any of the unsupported parameters
3878          */
3879         if (key ||
3880             (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP))
3881                 return -EOPNOTSUPP;
3882 
3883         if (!indir)
3884                 return 0;
3885 
3886         memcpy(pp->indir, indir, MVNETA_RSS_LU_TABLE_SIZE);
3887 
3888         return mvneta_config_rss(pp);
3889 }
3890 
3891 static int mvneta_ethtool_get_rxfh(struct net_device *dev, u32 *indir, u8 *key,
3892                                    u8 *hfunc)
3893 {
3894         struct mvneta_port *pp = netdev_priv(dev);
3895 
3896         /* Current code for Armada 3700 doesn't support RSS features yet */
3897         if (pp->neta_armada3700)
3898                 return -EOPNOTSUPP;
3899 
3900         if (hfunc)
3901                 *hfunc = ETH_RSS_HASH_TOP;
3902 
3903         if (!indir)
3904                 return 0;
3905 
3906         memcpy(indir, pp->indir, MVNETA_RSS_LU_TABLE_SIZE);
3907 
3908         return 0;
3909 }
3910 
3911 static const struct net_device_ops mvneta_netdev_ops = {
3912         .ndo_open            = mvneta_open,
3913         .ndo_stop            = mvneta_stop,
3914         .ndo_start_xmit      = mvneta_tx,
3915         .ndo_set_rx_mode     = mvneta_set_rx_mode,
3916         .ndo_set_mac_address = mvneta_set_mac_addr,
3917         .ndo_change_mtu      = mvneta_change_mtu,
3918         .ndo_fix_features    = mvneta_fix_features,
3919         .ndo_get_stats64     = mvneta_get_stats64,
3920         .ndo_do_ioctl        = mvneta_ioctl,
3921 };
3922 
3923 const struct ethtool_ops mvneta_eth_tool_ops = {
3924         .nway_reset     = phy_ethtool_nway_reset,
3925         .get_link       = ethtool_op_get_link,
3926         .set_coalesce   = mvneta_ethtool_set_coalesce,
3927         .get_coalesce   = mvneta_ethtool_get_coalesce,
3928         .get_drvinfo    = mvneta_ethtool_get_drvinfo,
3929         .get_ringparam  = mvneta_ethtool_get_ringparam,
3930         .set_ringparam  = mvneta_ethtool_set_ringparam,
3931         .get_strings    = mvneta_ethtool_get_strings,
3932         .get_ethtool_stats = mvneta_ethtool_get_stats,
3933         .get_sset_count = mvneta_ethtool_get_sset_count,
3934         .get_rxfh_indir_size = mvneta_ethtool_get_rxfh_indir_size,
3935         .get_rxnfc      = mvneta_ethtool_get_rxnfc,
3936         .get_rxfh       = mvneta_ethtool_get_rxfh,
3937         .set_rxfh       = mvneta_ethtool_set_rxfh,
3938         .get_link_ksettings = phy_ethtool_get_link_ksettings,
3939         .set_link_ksettings = mvneta_ethtool_set_link_ksettings,
3940 };
3941 
3942 /* Initialize hw */
3943 static int mvneta_init(struct device *dev, struct mvneta_port *pp)
3944 {
3945         int queue;
3946 
3947         /* Disable port */
3948         mvneta_port_disable(pp);
3949 
3950         /* Set port default values */
3951         mvneta_defaults_set(pp);
3952 
3953         pp->txqs = devm_kcalloc(dev, txq_number, sizeof(struct mvneta_tx_queue),
3954                                 GFP_KERNEL);
3955         if (!pp->txqs)
3956                 return -ENOMEM;
3957 
3958         /* Initialize TX descriptor rings */
3959         for (queue = 0; queue < txq_number; queue++) {
3960                 struct mvneta_tx_queue *txq = &pp->txqs[queue];
3961                 txq->id = queue;
3962                 txq->size = pp->tx_ring_size;
3963                 txq->done_pkts_coal = MVNETA_TXDONE_COAL_PKTS;
3964         }
3965 
3966         pp->rxqs = devm_kcalloc(dev, rxq_number, sizeof(struct mvneta_rx_queue),
3967                                 GFP_KERNEL);
3968         if (!pp->rxqs)
3969                 return -ENOMEM;
3970 
3971         /* Create Rx descriptor rings */
3972         for (queue = 0; queue < rxq_number; queue++) {
3973                 struct mvneta_rx_queue *rxq = &pp->rxqs[queue];
3974                 rxq->id = queue;
3975                 rxq->size = pp->rx_ring_size;
3976                 rxq->pkts_coal = MVNETA_RX_COAL_PKTS;
3977                 rxq->time_coal = MVNETA_RX_COAL_USEC;
3978                 rxq->buf_virt_addr = devm_kmalloc(pp->dev->dev.parent,
3979                                                   rxq->size * sizeof(void *),
3980                                                   GFP_KERNEL);
3981                 if (!rxq->buf_virt_addr)
3982                         return -ENOMEM;
3983         }
3984 
3985         return 0;
3986 }
3987 
3988 /* platform glue : initialize decoding windows */
3989 static void mvneta_conf_mbus_windows(struct mvneta_port *pp,
3990                                      const struct mbus_dram_target_info *dram)
3991 {
3992         u32 win_enable;
3993         u32 win_protect;
3994         int i;
3995 
3996         for (i = 0; i < 6; i++) {
3997                 mvreg_write(pp, MVNETA_WIN_BASE(i), 0);
3998                 mvreg_write(pp, MVNETA_WIN_SIZE(i), 0);
3999 
4000                 if (i < 4)
4001                         mvreg_write(pp, MVNETA_WIN_REMAP(i), 0);
4002         }
4003 
4004         win_enable = 0x3f;
4005         win_protect = 0;
4006 
4007         if (dram) {
4008                 for (i = 0; i < dram->num_cs; i++) {
4009                         const struct mbus_dram_window *cs = dram->cs + i;
4010 
4011                         mvreg_write(pp, MVNETA_WIN_BASE(i),
4012                                     (cs->base & 0xffff0000) |
4013                                     (cs->mbus_attr << 8) |
4014                                     dram->mbus_dram_target_id);
4015 
4016                         mvreg_write(pp, MVNETA_WIN_SIZE(i),
4017                                     (cs->size - 1) & 0xffff0000);
4018 
4019                         win_enable &= ~(1 << i);
4020                         win_protect |= 3 << (2 * i);
4021                 }
4022         } else {
4023                 /* For Armada3700 open default 4GB Mbus window, leaving
4024                  * arbitration of target/attribute to a different layer
4025                  * of configuration.
4026                  */
4027                 mvreg_write(pp, MVNETA_WIN_SIZE(0), 0xffff0000);
4028                 win_enable &= ~BIT(0);
4029                 win_protect = 3;
4030         }
4031 
4032         mvreg_write(pp, MVNETA_BASE_ADDR_ENABLE, win_enable);
4033         mvreg_write(pp, MVNETA_ACCESS_PROTECT_ENABLE, win_protect);
4034 }
4035 
4036 /* Power up the port */
4037 static int mvneta_port_power_up(struct mvneta_port *pp, int phy_mode)
4038 {
4039         u32 ctrl;
4040 
4041         /* MAC Cause register should be cleared */
4042         mvreg_write(pp, MVNETA_UNIT_INTR_CAUSE, 0);
4043 
4044         ctrl = mvreg_read(pp, MVNETA_GMAC_CTRL_2);
4045 
4046         /* Even though it might look weird, when we're configured in
4047          * SGMII or QSGMII mode, the RGMII bit needs to be set.
4048          */
4049         switch(phy_mode) {
4050         case PHY_INTERFACE_MODE_QSGMII:
4051                 mvreg_write(pp, MVNETA_SERDES_CFG, MVNETA_QSGMII_SERDES_PROTO);
4052                 ctrl |= MVNETA_GMAC2_PCS_ENABLE | MVNETA_GMAC2_PORT_RGMII;
4053                 break;
4054         case PHY_INTERFACE_MODE_SGMII:
4055                 mvreg_write(pp, MVNETA_SERDES_CFG, MVNETA_SGMII_SERDES_PROTO);
4056                 ctrl |= MVNETA_GMAC2_PCS_ENABLE | MVNETA_GMAC2_PORT_RGMII;
4057                 break;
4058         case PHY_INTERFACE_MODE_RGMII:
4059         case PHY_INTERFACE_MODE_RGMII_ID:
4060                 ctrl |= MVNETA_GMAC2_PORT_RGMII;
4061                 break;
4062         default:
4063                 return -EINVAL;
4064         }
4065 
4066         /* Cancel Port Reset */
4067         ctrl &= ~MVNETA_GMAC2_PORT_RESET;
4068         mvreg_write(pp, MVNETA_GMAC_CTRL_2, ctrl);
4069 
4070         while ((mvreg_read(pp, MVNETA_GMAC_CTRL_2) &
4071                 MVNETA_GMAC2_PORT_RESET) != 0)
4072                 continue;
4073 
4074         return 0;
4075 }
4076 
4077 /* Device initialization routine */
4078 static int mvneta_probe(struct platform_device *pdev)
4079 {
4080         const struct mbus_dram_target_info *dram_target_info;
4081         struct resource *res;
4082         struct device_node *dn = pdev->dev.of_node;
4083         struct device_node *phy_node;
4084         struct device_node *bm_node;
4085         struct mvneta_port *pp;
4086         struct net_device *dev;
4087         const char *dt_mac_addr;
4088         char hw_mac_addr[ETH_ALEN];
4089         const char *mac_from;
4090         const char *managed;
4091         int tx_csum_limit;
4092         int phy_mode;
4093         int err;
4094         int cpu;
4095 
4096         dev = alloc_etherdev_mqs(sizeof(struct mvneta_port), txq_number, rxq_number);
4097         if (!dev)
4098                 return -ENOMEM;
4099 
4100         dev->irq = irq_of_parse_and_map(dn, 0);
4101         if (dev->irq == 0) {
4102                 err = -EINVAL;
4103                 goto err_free_netdev;
4104         }
4105 
4106         phy_node = of_parse_phandle(dn, "phy", 0);
4107         if (!phy_node) {
4108                 if (!of_phy_is_fixed_link(dn)) {
4109                         dev_err(&pdev->dev, "no PHY specified\n");
4110                         err = -ENODEV;
4111                         goto err_free_irq;
4112                 }
4113 
4114                 err = of_phy_register_fixed_link(dn);
4115                 if (err < 0) {
4116                         dev_err(&pdev->dev, "cannot register fixed PHY\n");
4117                         goto err_free_irq;
4118                 }
4119 
4120                 /* In the case of a fixed PHY, the DT node associated
4121                  * to the PHY is the Ethernet MAC DT node.
4122                  */
4123                 phy_node = of_node_get(dn);
4124         }
4125 
4126         phy_mode = of_get_phy_mode(dn);
4127         if (phy_mode < 0) {
4128                 dev_err(&pdev->dev, "incorrect phy-mode\n");
4129                 err = -EINVAL;
4130                 goto err_put_phy_node;
4131         }
4132 
4133         dev->tx_queue_len = MVNETA_MAX_TXD;
4134         dev->watchdog_timeo = 5 * HZ;
4135         dev->netdev_ops = &mvneta_netdev_ops;
4136 
4137         dev->ethtool_ops = &mvneta_eth_tool_ops;
4138 
4139         pp = netdev_priv(dev);
4140         spin_lock_init(&pp->lock);
4141         pp->phy_node = phy_node;
4142         pp->phy_interface = phy_mode;
4143 
4144         err = of_property_read_string(dn, "managed", &managed);
4145         pp->use_inband_status = (err == 0 &&
4146                                  strcmp(managed, "in-band-status") == 0);
4147 
4148         pp->rxq_def = rxq_def;
4149 
4150         /* Set RX packet offset correction for platforms, whose
4151          * NET_SKB_PAD, exceeds 64B. It should be 64B for 64-bit
4152          * platforms and 0B for 32-bit ones.
4153          */
4154         pp->rx_offset_correction =
4155                 max(0, NET_SKB_PAD - MVNETA_RX_PKT_OFFSET_CORRECTION);
4156 
4157         pp->indir[0] = rxq_def;
4158 
4159         /* Get special SoC configurations */
4160         if (of_device_is_compatible(dn, "marvell,armada-3700-neta"))
4161                 pp->neta_armada3700 = true;
4162 
4163         pp->clk = devm_clk_get(&pdev->dev, "core");
4164         if (IS_ERR(pp->clk))
4165                 pp->clk = devm_clk_get(&pdev->dev, NULL);
4166         if (IS_ERR(pp->clk)) {
4167                 err = PTR_ERR(pp->clk);
4168                 goto err_put_phy_node;
4169         }
4170 
4171         clk_prepare_enable(pp->clk);
4172 
4173         pp->clk_bus = devm_clk_get(&pdev->dev, "bus");
4174         if (!IS_ERR(pp->clk_bus))
4175                 clk_prepare_enable(pp->clk_bus);
4176 
4177         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
4178         pp->base = devm_ioremap_resource(&pdev->dev, res);
4179         if (IS_ERR(pp->base)) {
4180                 err = PTR_ERR(pp->base);
4181                 goto err_clk;
4182         }
4183 
4184         /* Alloc per-cpu port structure */
4185         pp->ports = alloc_percpu(struct mvneta_pcpu_port);
4186         if (!pp->ports) {
4187                 err = -ENOMEM;
4188                 goto err_clk;
4189         }
4190 
4191         /* Alloc per-cpu stats */
4192         pp->stats = netdev_alloc_pcpu_stats(struct mvneta_pcpu_stats);
4193         if (!pp->stats) {
4194                 err = -ENOMEM;
4195                 goto err_free_ports;
4196         }
4197 
4198         dt_mac_addr = of_get_mac_address(dn);
4199         if (dt_mac_addr) {
4200                 mac_from = "device tree";
4201                 memcpy(dev->dev_addr, dt_mac_addr, ETH_ALEN);
4202         } else {
4203                 mvneta_get_mac_addr(pp, hw_mac_addr);
4204                 if (is_valid_ether_addr(hw_mac_addr)) {
4205                         mac_from = "hardware";
4206                         memcpy(dev->dev_addr, hw_mac_addr, ETH_ALEN);
4207                 } else {
4208                         mac_from = "random";
4209                         eth_hw_addr_random(dev);
4210                 }
4211         }
4212 
4213         if (!of_property_read_u32(dn, "tx-csum-limit", &tx_csum_limit)) {
4214                 if (tx_csum_limit < 0 ||
4215                     tx_csum_limit > MVNETA_TX_CSUM_MAX_SIZE) {
4216                         tx_csum_limit = MVNETA_TX_CSUM_DEF_SIZE;
4217                         dev_info(&pdev->dev,
4218                                  "Wrong TX csum limit in DT, set to %dB\n",
4219                                  MVNETA_TX_CSUM_DEF_SIZE);
4220                 }
4221         } else if (of_device_is_compatible(dn, "marvell,armada-370-neta")) {
4222                 tx_csum_limit = MVNETA_TX_CSUM_DEF_SIZE;
4223         } else {
4224                 tx_csum_limit = MVNETA_TX_CSUM_MAX_SIZE;
4225         }
4226 
4227         pp->tx_csum_limit = tx_csum_limit;
4228 
4229         dram_target_info = mv_mbus_dram_info();
4230         /* Armada3700 requires setting default configuration of Mbus
4231          * windows, however without using filled mbus_dram_target_info
4232          * structure.
4233          */
4234         if (dram_target_info || pp->neta_armada3700)
4235                 mvneta_conf_mbus_windows(pp, dram_target_info);
4236 
4237         pp->tx_ring_size = MVNETA_MAX_TXD;
4238         pp->rx_ring_size = MVNETA_MAX_RXD;
4239 
4240         pp->dev = dev;
4241         SET_NETDEV_DEV(dev, &pdev->dev);
4242 
4243         pp->id = global_port_id++;
4244 
4245         /* Obtain access to BM resources if enabled and already initialized */
4246         bm_node = of_parse_phandle(dn, "buffer-manager", 0);
4247         if (bm_node && bm_node->data) {
4248                 pp->bm_priv = bm_node->data;
4249                 err = mvneta_bm_port_init(pdev, pp);
4250                 if (err < 0) {
4251                         dev_info(&pdev->dev, "use SW buffer management\n");
4252                         pp->bm_priv = NULL;
4253                 }
4254         }
4255         of_node_put(bm_node);
4256 
4257         err = mvneta_init(&pdev->dev, pp);
4258         if (err < 0)
4259                 goto err_netdev;
4260 
4261         err = mvneta_port_power_up(pp, phy_mode);
4262         if (err < 0) {
4263                 dev_err(&pdev->dev, "can't power up port\n");
4264                 goto err_netdev;
4265         }
4266 
4267         /* Armada3700 network controller does not support per-cpu
4268          * operation, so only single NAPI should be initialized.
4269          */
4270         if (pp->neta_armada3700) {
4271                 netif_napi_add(dev, &pp->napi, mvneta_poll, NAPI_POLL_WEIGHT);
4272         } else {
4273                 for_each_present_cpu(cpu) {
4274                         struct mvneta_pcpu_port *port =
4275                                 per_cpu_ptr(pp->ports, cpu);
4276 
4277                         netif_napi_add(dev, &port->napi, mvneta_poll,
4278                                        NAPI_POLL_WEIGHT);
4279                         port->pp = pp;
4280                 }
4281         }
4282 
4283         dev->features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
4284         dev->hw_features |= dev->features;
4285         dev->vlan_features |= dev->features;
4286         dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
4287         dev->gso_max_segs = MVNETA_MAX_TSO_SEGS;
4288 
4289         /* MTU range: 68 - 9676 */
4290         dev->min_mtu = ETH_MIN_MTU;
4291         /* 9676 == 9700 - 20 and rounding to 8 */
4292         dev->max_mtu = 9676;
4293 
4294         err = register_netdev(dev);
4295         if (err < 0) {
4296                 dev_err(&pdev->dev, "failed to register\n");
4297                 goto err_free_stats;
4298         }
4299 
4300         netdev_info(dev, "Using %s mac address %pM\n", mac_from,
4301                     dev->dev_addr);
4302 
4303         platform_set_drvdata(pdev, pp->dev);
4304 
4305         if (pp->use_inband_status) {
4306                 struct phy_device *phy = of_phy_find_device(dn);
4307 
4308                 mvneta_fixed_link_update(pp, phy);
4309 
4310                 put_device(&phy->mdio.dev);
4311         }
4312 
4313         return 0;
4314 
4315 err_netdev:
4316         unregister_netdev(dev);
4317         if (pp->bm_priv) {
4318                 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id);
4319                 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_short,
4320                                        1 << pp->id);
4321         }
4322 err_free_stats:
4323         free_percpu(pp->stats);
4324 err_free_ports:
4325         free_percpu(pp->ports);
4326 err_clk:
4327         clk_disable_unprepare(pp->clk_bus);
4328         clk_disable_unprepare(pp->clk);
4329 err_put_phy_node:
4330         of_node_put(phy_node);
4331         if (of_phy_is_fixed_link(dn))
4332                 of_phy_deregister_fixed_link(dn);
4333 err_free_irq:
4334         irq_dispose_mapping(dev->irq);
4335 err_free_netdev:
4336         free_netdev(dev);
4337         return err;
4338 }
4339 
4340 /* Device removal routine */
4341 static int mvneta_remove(struct platform_device *pdev)
4342 {
4343         struct net_device  *dev = platform_get_drvdata(pdev);
4344         struct device_node *dn = pdev->dev.of_node;
4345         struct mvneta_port *pp = netdev_priv(dev);
4346 
4347         unregister_netdev(dev);
4348         clk_disable_unprepare(pp->clk_bus);
4349         clk_disable_unprepare(pp->clk);
4350         free_percpu(pp->ports);
4351         free_percpu(pp->stats);
4352         if (of_phy_is_fixed_link(dn))
4353                 of_phy_deregister_fixed_link(dn);
4354         irq_dispose_mapping(dev->irq);
4355         of_node_put(pp->phy_node);
4356         free_netdev(dev);
4357 
4358         if (pp->bm_priv) {
4359                 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id);
4360                 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_short,
4361                                        1 << pp->id);
4362         }
4363 
4364         return 0;
4365 }
4366 
4367 static const struct of_device_id mvneta_match[] = {
4368         { .compatible = "marvell,armada-370-neta" },
4369         { .compatible = "marvell,armada-xp-neta" },
4370         { .compatible = "marvell,armada-3700-neta" },
4371         { }
4372 };
4373 MODULE_DEVICE_TABLE(of, mvneta_match);
4374 
4375 static struct platform_driver mvneta_driver = {
4376         .probe = mvneta_probe,
4377         .remove = mvneta_remove,
4378         .driver = {
4379                 .name = MVNETA_DRIVER_NAME,
4380                 .of_match_table = mvneta_match,
4381         },
4382 };
4383 
4384 static int __init mvneta_driver_init(void)
4385 {
4386         int ret;
4387 
4388         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "net/mvmeta:online",
4389                                       mvneta_cpu_online,
4390                                       mvneta_cpu_down_prepare);
4391         if (ret < 0)
4392                 goto out;
4393         online_hpstate = ret;
4394         ret = cpuhp_setup_state_multi(CPUHP_NET_MVNETA_DEAD, "net/mvneta:dead",
4395                                       NULL, mvneta_cpu_dead);
4396         if (ret)
4397                 goto err_dead;
4398 
4399         ret = platform_driver_register(&mvneta_driver);
4400         if (ret)
4401                 goto err;
4402         return 0;
4403 
4404 err:
4405         cpuhp_remove_multi_state(CPUHP_NET_MVNETA_DEAD);
4406 err_dead:
4407         cpuhp_remove_multi_state(online_hpstate);
4408 out:
4409         return ret;
4410 }
4411 module_init(mvneta_driver_init);
4412 
4413 static void __exit mvneta_driver_exit(void)
4414 {
4415         platform_driver_unregister(&mvneta_driver);
4416         cpuhp_remove_multi_state(CPUHP_NET_MVNETA_DEAD);
4417         cpuhp_remove_multi_state(online_hpstate);
4418 }
4419 module_exit(mvneta_driver_exit);
4420 
4421 MODULE_DESCRIPTION("Marvell NETA Ethernet Driver - www.marvell.com");
4422 MODULE_AUTHOR("Rami Rosen <rosenr@marvell.com>, Thomas Petazzoni <thomas.petazzoni@free-electrons.com>");
4423 MODULE_LICENSE("GPL");
4424 
4425 module_param(rxq_number, int, S_IRUGO);
4426 module_param(txq_number, int, S_IRUGO);
4427 
4428 module_param(rxq_def, int, S_IRUGO);
4429 module_param(rx_copybreak, int, S_IRUGO | S_IWUSR);
4430 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us