bcm-knet.c (292120B)
1 /* 2 * Copyright 2017 Broadcom 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License, version 2, as 6 * published by the Free Software Foundation (the "GPL"). 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License version 2 (GPLv2) for more details. 12 * 13 * You should have received a copy of the GNU General Public License 14 * version 2 (GPLv2) along with this source code. 15 */ 16 17 /* 18 * This module implements a Linux network driver for Broadcom 19 * XGS switch devices. The driver simultaneously serves a 20 * number of vitual Linux network devices and a Tx/Rx API 21 * implemented in user space. 22 * 23 * Packets received from the switch device are sent to either 24 * a virtual Linux network device or the user mode Rx API 25 * based on a set of packet filters.susp 26 * 27 * Packets from the virtual Linux network devices and the user 28 * mode Tx API are multiplexed with priority given to the Tx API. 29 * 30 * A message-based IOCTL interface is used for managing packet 31 * filters and virtual Linux network interfaces. 32 * 33 * A virtual network interface can be configured to work in RCPU 34 * mode, which means that packets from the switch device will 35 * be encasulated with a RCPU header and a block of meta data 36 * that basically contains the core DCB information. Likewise, 37 * packets received from the Linux network stack are assumed to 38 * be RCPU encapsulated when going out on an interface in RCPU 39 * mode. 40 * 41 * The module implements basic Rx DMA rate control. The rate is 42 * specified in packets per second, and different Rx DMA channels 43 * can be configured to use different maximum packet rates. 44 * The packet rate can be configure as a module parameter, and 45 * it can also be changed dynamically through the proc file 46 * system (syntax is described in function header comment). 47 * 48 * To support multiple instance, each instance has its event queue. 49 * 50 * To support pci hot-plug in this module, the resource update 51 * should be handled when the PCI device is re-plugged. 52 * NOTE: the KNET detach should be invoked before removing the 53 * device. 54 * 55 * For a list of supported module parameters, please see below. 56 */ 57 58 #include <gmodule.h> /* Must be included first */ 59 #include <linux-bde.h> 60 #include <kcom.h> 61 #include <bcm-knet.h> 62 63 #include <linux/netdevice.h> 64 #include <linux/etherdevice.h> 65 #include <linux/ethtool.h> 66 #include <linux/random.h> 67 #include <linux/seq_file.h> 68 #include <linux/if_vlan.h> 69 #include <linux/nsproxy.h> 70 71 72 MODULE_AUTHOR("Broadcom Corporation"); 73 MODULE_DESCRIPTION("Network Device Driver for Broadcom BCM TxRx API"); 74 MODULE_LICENSE("GPL"); 75 76 static int debug; 77 LKM_MOD_PARAM(debug, "i", int, 0); 78 MODULE_PARM_DESC(debug, 79 "Debug level (default 0)"); 80 81 static char *mac_addr = NULL; 82 LKM_MOD_PARAM(mac_addr, "s", charp, 0); 83 MODULE_PARM_DESC(mac_addr, 84 "Ethernet MAC address (default 02:10:18:xx:xx:xx)"); 85 86 static int rx_buffer_size = 9216; 87 LKM_MOD_PARAM(rx_buffer_size, "i", int, 0); 88 MODULE_PARM_DESC(rx_buffer_size, 89 "Size of RX packet buffers (default 9216)"); 90 91 static int default_mtu = 1500; 92 LKM_MOD_PARAM(default_mtu, "i", int, 0); 93 MODULE_PARM_DESC(default_mtu, 94 "Default MTU for KNET network interfaces (default 1500)"); 95 96 static int rx_sync_retry = 1000; 97 LKM_MOD_PARAM(rx_sync_retry, "i", int, 0); 98 MODULE_PARM_DESC(rx_sync_retry, 99 "Retries if chain is incomplete on interrupt (default 10)"); 100 101 static char *base_dev_name = NULL; 102 LKM_MOD_PARAM(base_dev_name, "s", charp, 0); 103 MODULE_PARM_DESC(base_dev_name, 104 "Base device name (default bcm0, bcm1, etc.)"); 105 106 static int rcpu_mode = 0; 107 LKM_MOD_PARAM(rcpu_mode, "i", int, 0); 108 MODULE_PARM_DESC(rcpu_mode, 109 "Enable RCPU encapsulation (default 0)"); 110 111 static char *rcpu_dmac = NULL; 112 LKM_MOD_PARAM(rcpu_dmac, "s", charp, 0); 113 MODULE_PARM_DESC(rcpu_dmac, 114 "RCPU destination MAC address (by default use L2 destination MAC address)"); 115 116 static char *rcpu_smac = NULL; 117 LKM_MOD_PARAM(rcpu_smac, "s", charp, 0); 118 MODULE_PARM_DESC(rcpu_smac, 119 "RCPU source MAC address (by default use L2 source MAC address)"); 120 121 static int rcpu_ethertype = 0xde08; 122 LKM_MOD_PARAM(rcpu_ethertype, "i", int, 0); 123 MODULE_PARM_DESC(rcpu_ethertype, 124 "RCPU EtherType (default DE08h)"); 125 126 static int rcpu_signature = 0; 127 LKM_MOD_PARAM(rcpu_signature, "i", int, 0); 128 MODULE_PARM_DESC(rcpu_signature, 129 "RCPU Signature (default is PCI device ID)"); 130 131 static int rcpu_vlan = 1; 132 LKM_MOD_PARAM(rcpu_vlan, "i", int, 0); 133 MODULE_PARM_DESC(rcpu_vlan, 134 "RCPU VLAN ID (default 1)"); 135 136 static int use_rx_skb = 0; 137 LKM_MOD_PARAM(use_rx_skb, "i", int, 0); 138 MODULE_PARM_DESC(use_rx_skb, 139 "Use socket buffers for receive operation (default 0)"); 140 141 static int num_rx_prio = 1; 142 LKM_MOD_PARAM(num_rx_prio, "i", int, 0); 143 MODULE_PARM_DESC(num_rx_prio, 144 "Number of filter priorities per Rx DMA channel"); 145 146 static int rx_rate[8] = { 100000, 100000, 100000, 100000, 100000, 100000, 100000, 0 }; 147 LKM_MOD_PARAM_ARRAY(rx_rate, "1-4i", int, NULL, 0); 148 MODULE_PARM_DESC(rx_rate, 149 "Rx rate in packets per second (default 100000)"); 150 151 static int rx_burst[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; 152 LKM_MOD_PARAM_ARRAY(rx_burst, "1-4i", int, NULL, 0); 153 MODULE_PARM_DESC(rx_burst, 154 "Rx rate burst maximum in packets (default rx_rate/10)"); 155 156 static int check_rcpu_signature = 0; 157 LKM_MOD_PARAM(check_rcpu_signature, "i", int, 0); 158 MODULE_PARM_DESC(check_rcpu_signature, 159 "Check RCPU Signature for Tx packets from RCPU interfaces"); 160 161 static int basedev_suspend = 0; 162 LKM_MOD_PARAM(basedev_suspend, "i", int, 0); 163 MODULE_PARM_DESC(basedev_suspend, 164 "Pause traffic till base device is up (enabled by default in NAPI mode)"); 165 166 /* 167 * Force to add one layer of VLAN tag to untagged packets on Dune devices 168 */ 169 static int force_tagged = 1; 170 LKM_MOD_PARAM(force_tagged, "i", int, 0); 171 MODULE_PARM_DESC(force_tagged, 172 "Always tagged with VLAN tag with spceified VID or VSI(default 1)"); 173 174 static int ft_tpid=0x8100; 175 LKM_MOD_PARAM(ft_tpid, "i", int, 0); 176 MODULE_PARM_DESC(ft_tpid, 177 "Tag Protocol Identifier (TPID) indicates the frame type (default 0x8100)"); 178 179 static int ft_pri=0; 180 LKM_MOD_PARAM(ft_pri, "i", int, 0); 181 MODULE_PARM_DESC(ft_cfi, 182 "Priority (PRI) indicates the frame priority (default 0)"); 183 184 static int ft_cfi=0; 185 LKM_MOD_PARAM(ft_cfi, "i", int, 0); 186 MODULE_PARM_DESC(ft_cfi, 187 "Canonical Format Indicator (CFI) indicates whether a MAC address is encapsulated in canonical format over different transmission media (default 0)"); 188 189 static int ft_vid=0; 190 LKM_MOD_PARAM(ft_vid, "i", int, 0); 191 MODULE_PARM_DESC(ft_vid, 192 "VLAN ID (VID) indicates the VLAN to which a frame belongs (default 0)"); 193 194 195 /* Debug levels */ 196 #define DBG_LVL_VERB 0x1 197 #define DBG_LVL_DCB 0x2 198 #define DBG_LVL_PKT 0x4 199 #define DBG_LVL_SKB 0x8 200 #define DBG_LVL_CMD 0x10 201 #define DBG_LVL_EVT 0x20 202 #define DBG_LVL_IRQ 0x40 203 #define DBG_LVL_NAPI 0x80 204 #define DBG_LVL_PDMP 0x100 205 #define DBG_LVL_FLTR 0x200 206 #define DBG_LVL_KCOM 0x400 207 #define DBG_LVL_RCPU 0x800 208 #define DBG_LVL_WARN 0x1000 209 #define DBG_LVL_NDEV 0x2000 210 #define DBG_LVL_INST 0x4000 211 /* Level to output Dune internal headers Parsing */ 212 #define DBG_LVL_DUNE 0x8000 213 #define DBG_LVL_DCB_TX 0x10000 214 #define DBG_LVL_DCB_RX 0x20000 215 #define DBG_LVL_PDMP_TX 0x40000 216 #define DBG_LVL_PDMP_RX 0x80000 217 218 #define DBG_VERB(_s) do { if (debug & DBG_LVL_VERB) gprintk _s; } while (0) 219 #define DBG_PKT(_s) do { if (debug & DBG_LVL_PKT) gprintk _s; } while (0) 220 #define DBG_SKB(_s) do { if (debug & DBG_LVL_SKB) gprintk _s; } while (0) 221 #define DBG_CMD(_s) do { if (debug & DBG_LVL_CMD) gprintk _s; } while (0) 222 #define DBG_EVT(_s) do { if (debug & DBG_LVL_EVT) gprintk _s; } while (0) 223 #define DBG_IRQ(_s) do { if (debug & DBG_LVL_IRQ) gprintk _s; } while (0) 224 #define DBG_NAPI(_s) do { if (debug & DBG_LVL_NAPI) gprintk _s; } while (0) 225 #define DBG_PDMP(_s) do { if (debug & DBG_LVL_PDMP) gprintk _s; } while (0) 226 #define DBG_FLTR(_s) do { if (debug & DBG_LVL_FLTR) gprintk _s; } while (0) 227 #define DBG_KCOM(_s) do { if (debug & DBG_LVL_KCOM) gprintk _s; } while (0) 228 #define DBG_RCPU(_s) do { if (debug & DBG_LVL_RCPU) gprintk _s; } while (0) 229 #define DBG_WARN(_s) do { if (debug & DBG_LVL_WARN) gprintk _s; } while (0) 230 #define DBG_NDEV(_s) do { if (debug & DBG_LVL_NDEV) gprintk _s; } while (0) 231 #define DBG_INST(_s) do { if (debug & DBG_LVL_INST) gprintk _s; } while (0) 232 #define DBG_DUNE(_s) do { if (debug & DBG_LVL_DUNE) gprintk _s; } while (0) 233 #define DBG_DCB_TX(_s) do { if (debug & (DBG_LVL_DCB|DBG_LVL_DCB_TX)) \ 234 gprintk _s; } while (0) 235 #define DBG_DCB_RX(_s) do { if (debug & (DBG_LVL_DCB|DBG_LVL_DCB_RX)) \ 236 gprintk _s; } while (0) 237 #define DBG_DCB(_s) do { if (debug & (DBG_LVL_DCB|DBG_LVL_DCB_TX| \ 238 DBG_LVL_DCB_RX)) \ 239 gprintk _s; } while (0) 240 241 242 /* This flag is used to indicate if debugging packet function is open or closed */ 243 static int dbg_pkt_enable = 0; 244 245 /* Module Information */ 246 #define MODULE_MAJOR 122 247 #define MODULE_NAME "linux-bcm-knet" 248 249 #ifndef NAPI_SUPPORT 250 #define NAPI_SUPPORT 1 251 #endif 252 253 #if NAPI_SUPPORT 254 255 static int use_napi = 0; 256 LKM_MOD_PARAM(use_napi, "i", int, 0); 257 MODULE_PARM_DESC(use_napi, 258 "Use NAPI interface (default 0)"); 259 260 static int napi_weight = 64; 261 LKM_MOD_PARAM(napi_weight, "i", int, 0); 262 MODULE_PARM_DESC(napi_weight, 263 "Weight of NAPI interfaces (default 64)"); 264 265 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) 266 #define bkn_napi_enable(_dev, _napi) netif_poll_enable(_dev) 267 #define bkn_napi_disable(_dev, _napi) netif_poll_disable(_dev) 268 #define bkn_napi_schedule(_dev, _napi) netif_rx_schedule(_dev) 269 #define bkn_napi_schedule_prep(_dev, _napi) netif_rx_schedule_prep(_dev) 270 #define __bkn_napi_schedule(_dev, _napi) __netif_rx_schedule(_dev) 271 #define bkn_napi_complete(_dev, _napi) netif_rx_complete(_dev) 272 #else 273 #define bkn_napi_enable(_dev, _napi) napi_enable(_napi) 274 #define bkn_napi_disable(_dev, _napi) napi_disable(_napi) 275 #define bkn_napi_schedule(_dev, _napi) napi_schedule(_napi) 276 #define bkn_napi_schedule_prep(_dev, _napi) napi_schedule_prep(_napi) 277 #define __bkn_napi_schedule(_dev, _napi) __napi_schedule(_napi) 278 #define bkn_napi_complete(_dev, _napi) napi_complete(_napi) 279 #endif 280 281 #else 282 283 static int use_napi = 0; 284 static int napi_weight = 0; 285 286 #define bkn_napi_enable(_dev, _napi) 287 #define bkn_napi_disable(_dev, _napi) 288 #define bkn_napi_schedule(_dev, _napi) 289 #define bkn_napi_schedule_prep(_dev, _napi) (0) 290 #define __bkn_napi_schedule(_dev, _napi) 291 #define bkn_napi_complete(_dev, _napi) 292 293 #endif 294 295 /* Compatibility */ 296 297 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4,7,0)) 298 #define NETDEV_UPDATE_TRANS_START_TIME(dev) dev->trans_start = jiffies 299 #else 300 #define NETDEV_UPDATE_TRANS_START_TIME(dev) netif_trans_update(dev) 301 #endif 302 303 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24)) 304 #define skb_copy_to_linear_data(_skb, _pkt, _len) \ 305 eth_copy_and_sum(_skb, _pkt, _len, 0) 306 struct napi_struct { int not_used; }; 307 #define netif_napi_add(_dev, _napi, _poll, _weight) do { \ 308 (_dev)->poll = _poll; \ 309 (_dev)->weight = _weight; \ 310 } while(0) 311 #endif 312 313 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18)) 314 #define SKB_PADTO(_skb,_len) (((_skb = skb_padto(_skb,_len)) == NULL) ? -1 : 0) 315 #else 316 #define SKB_PADTO(_skb,_len) skb_padto(_skb,_len) 317 #endif 318 319 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,12)) 320 #define skb_header_cloned(_skb) \ 321 skb_cloned(_skb) 322 #endif 323 324 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,4,27) 325 static inline void *netdev_priv(struct net_device *dev) 326 { 327 return dev->priv; 328 } 329 #endif /* KERNEL_VERSION(2,4,27) */ 330 331 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,4,23) 332 /* Special check for MontaVista 2.4.20 MIPS */ 333 #if !(defined(MAX_USER_RT_PRIO) && defined(CONFIG_MIPS)) 334 static inline void free_netdev(struct net_device *dev) 335 { 336 kfree(dev); 337 } 338 #endif 339 static inline void netif_poll_disable(struct net_device *dev) 340 { 341 while (test_and_set_bit(__LINK_STATE_RX_SCHED, &dev->state)) { 342 /* No hurry. */ 343 current->state = TASK_INTERRUPTIBLE; 344 schedule_timeout(1); 345 } 346 } 347 static inline void netif_poll_enable(struct net_device *dev) 348 { 349 clear_bit(__LINK_STATE_RX_SCHED, &dev->state); 350 } 351 #endif /* KERNEL_VERSION(2,4,23) */ 352 353 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,4,21) 354 static struct sk_buff *skb_pad(struct sk_buff *skb, int pad) 355 { 356 struct sk_buff *nskb; 357 358 /* If the skbuff is non linear tailroom is always zero.. */ 359 if(skb_tailroom(skb) >= pad) 360 { 361 memset(skb->data+skb->len, 0, pad); 362 return skb; 363 } 364 365 nskb = skb_copy_expand(skb, skb_headroom(skb), skb_tailroom(skb) + pad, GFP_ATOMIC); 366 kfree_skb(skb); 367 if(nskb) 368 memset(nskb->data+nskb->len, 0, pad); 369 return nskb; 370 } 371 static inline struct sk_buff *skb_padto(struct sk_buff *skb, unsigned int len) 372 { 373 unsigned int size = skb->len; 374 if(likely(size >= len)) 375 return skb; 376 return skb_pad(skb, len-size); 377 } 378 #endif /* KERNEL_VERSION(2,4,21) */ 379 380 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,10,0) 381 #define bkn_vlan_hwaccel_put_tag(_skb, _proto, _tci) \ 382 __vlan_hwaccel_put_tag(_skb, _tci) 383 #else 384 #define bkn_vlan_hwaccel_put_tag(_skb, _proto, _tci) \ 385 __vlan_hwaccel_put_tag(_skb, htons(_proto), _tci) 386 #endif 387 388 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,1,0) 389 #define ETH_P_8021AD 0x88A8 /* 802.1ad Service VLAN */ 390 #endif 391 392 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,27) 393 #define bkn_dma_mapping_error(d, a) \ 394 dma_mapping_error(a) 395 #define bkn_pci_dma_mapping_error(d, a) \ 396 pci_dma_mapping_error(a) 397 #else 398 #define bkn_dma_mapping_error(d, a) \ 399 dma_mapping_error(d, a) 400 #define bkn_pci_dma_mapping_error(d, a) \ 401 pci_dma_mapping_error(d, a) 402 #endif 403 404 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,30) 405 enum hwtstamp_tx_types { 406 HWTSTAMP_TX_OFF, 407 HWTSTAMP_TX_ON, 408 HWTSTAMP_TX_ONESTEP_SYNC 409 }; 410 enum { 411 SKBTX_HW_TSTAMP = 1 << 0, 412 SKBTX_SW_TSTAMP = 1 << 1, 413 SKBTX_IN_PROGRESS = 1 << 2, 414 }; 415 struct skb_shared_hwtstamps { 416 ktime_t hwtstamp; 417 ktime_t syststamp; 418 }; 419 struct bkn_skb_shared_info { 420 uint8_t tx_flags; 421 struct skb_shared_hwtstamps hwtstamps; 422 }; 423 #define bkn_skb_shinfo(_skb) ((struct bkn_skb_shared_info *)(unsigned char *)_skb->end) 424 #define bkn_skb_tx_flags(_skb) bkn_skb_shinfo(_skb)->tx_flags 425 static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb) 426 { 427 return &bkn_skb_shinfo(skb)->hwtstamps; 428 } 429 void skb_tstamp_tx(struct sk_buff *orig_skb, struct skb_shared_hwtstamps *hwtstamps) 430 { 431 } 432 static inline void bkn_skb_tx_timestamp(struct sk_buff *skb) 433 { 434 } 435 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,26) 436 static inline ktime_t ns_to_ktime(u64 ns) 437 { 438 static const ktime_t ktime; 439 return ktime; 440 } 441 #endif 442 #elif LINUX_VERSION_CODE < KERNEL_VERSION(2,6,37) 443 #include <linux/net_tstamp.h> 444 #define HWTSTAMP_TX_ONESTEP_SYNC 2 445 enum { 446 SKBTX_HW_TSTAMP = 1 << 0, 447 SKBTX_SW_TSTAMP = 1 << 1, 448 SKBTX_IN_PROGRESS = 1 << 2, 449 }; 450 #define bkn_skb_tx_flags(_skb) skb_shinfo(_skb)->tx_flags.flags 451 static inline void bkn_skb_tx_timestamp(struct sk_buff *skb) 452 { 453 } 454 #else 455 #include <linux/net_tstamp.h> 456 457 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,2,0) 458 #define HWTSTAMP_TX_ONESTEP_SYNC 2 459 #endif 460 461 #define bkn_skb_tx_flags(_skb) skb_shinfo(_skb)->tx_flags 462 static inline void bkn_skb_tx_timestamp(struct sk_buff *skb) 463 { 464 return skb_tx_timestamp(skb); 465 } 466 #endif 467 468 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,26) 469 #define bkn_dev_net_set(dev, net) 470 #else 471 #define bkn_dev_net_set(dev, net) dev_net_set(dev, net) 472 #endif 473 474 #ifdef LINUX_BDE_DMA_DEVICE_SUPPORT 475 #define BKN_DMA_DEV device 476 #define BKN_DMA_FROMDEV DMA_FROM_DEVICE 477 #define BKN_DMA_TODEV DMA_TO_DEVICE 478 #define BKN_DMA_MAP_SINGLE(d,p,s,r) dma_map_single(d,p,s,r) 479 #define BKN_DMA_UNMAP_SINGLE(d,a,s,r) dma_unmap_single(d,a,s,r) 480 #define BKN_DMA_ALLOC_COHERENT(d,s,h) dma_alloc_coherent(d,s,h,GFP_ATOMIC|GFP_DMA32) 481 #define BKN_DMA_FREE_COHERENT(d,s,a,h) dma_free_coherent(d,s,a,h) 482 #define BKN_DMA_MAPPING_ERROR(d,a) bkn_dma_mapping_error(d,a) 483 #else 484 #define BKN_DMA_DEV pci_dev 485 #define BKN_DMA_FROMDEV PCI_DMA_FROMDEVICE 486 #define BKN_DMA_TODEV PCI_DMA_TODEVICE 487 #define BKN_DMA_MAP_SINGLE(d,p,s,r) pci_map_single(d,p,s,r) 488 #define BKN_DMA_UNMAP_SINGLE(d,a,s,r) pci_unmap_single(d,a,s,r) 489 #define BKN_DMA_ALLOC_COHERENT(d,s,h) pci_alloc_consistent(d,s,h) 490 #define BKN_DMA_FREE_COHERENT(d,s,a,h) pci_free_consistent(d,s,a,h) 491 #define BKN_DMA_MAPPING_ERROR(d,a) bkn_pci_dma_mapping_error(d,a) 492 #endif 493 494 /* 495 * Get a 16-bit value from packet offset 496 * _data Pointer to packet 497 * _offset Offset 498 */ 499 #define PKT_U16_GET(_data, _offset) \ 500 (uint16_t)(_data[_offset] << 8 | _data[_offset + 1]) 501 502 503 /* RCPU operations */ 504 #define RCPU_OPCODE_RX 0x10 505 #define RCPU_OPCODE_TX 0x20 506 507 /* RCPU flags */ 508 #define RCPU_F_MODHDR 0x4 509 510 /* RCPU encapsulation */ 511 #define RCPU_HDR_SIZE 32 512 #define RCPU_TX_META_SIZE 32 513 #define RCPU_TX_ENCAP_SIZE (RCPU_HDR_SIZE + RCPU_TX_META_SIZE) 514 #define RCPU_RX_META_SIZE 64 515 #define RCPU_RX_ENCAP_SIZE (RCPU_HDR_SIZE + RCPU_RX_META_SIZE) 516 517 #define PKT_TX_HDR_SIZE 16 518 519 static volatile int module_initialized; 520 521 static ibde_t *kernel_bde = NULL; 522 523 /* Descriptor info */ 524 typedef struct bkn_desc_info_s { 525 uint32_t *dcb_mem; 526 uint64_t dcb_dma; 527 struct sk_buff *skb; 528 uint64_t skb_dma; 529 uint32_t dma_size; 530 } bkn_desc_info_t; 531 532 /* DCB chain info */ 533 typedef struct bkn_dcb_chain_s { 534 struct list_head list; 535 int dcb_cnt; 536 int dcb_cur; 537 uint32_t *dcb_mem; 538 uint64_t dcb_dma; 539 } bkn_dcb_chain_t; 540 541 #define MAX_TX_DCBS 64 542 #define MAX_RX_DCBS 64 543 544 #define NUM_DMA_CHAN 8 545 #define NUM_RX_CHAN 7 546 #define NUM_CMICX_RX_CHAN 7 547 #define NUM_CMICM_RX_CHAN 3 548 549 #define FCS_SZ 4 550 #define TAG_SZ 4 551 552 /* Device control info */ 553 typedef struct bkn_switch_info_s { 554 struct list_head list; 555 struct list_head ndev_list; /* Associated virtual Ethernet interfaces */ 556 struct net_device **ndevs; /* Indexed array of ndev_list */ 557 int ndev_max; /* Size of indexed array */ 558 struct list_head rxpf_list; /* Associated Rx packet filters */ 559 volatile void *base_addr; /* Base address for PCI register access */ 560 struct BKN_DMA_DEV *dma_dev; /* Required for DMA memory control */ 561 struct pci_dev *pdev; /* Required for DMA memory control */ 562 struct net_device *dev; /* Base network device */ 563 struct napi_struct napi; /* New NAPI */ 564 struct timer_list timer; /* Retry/resource timer */ 565 int timer_queued; /* Flag indicating queued timer function */ 566 uint32_t timer_runs; /* Timer function runs (debug only) */ 567 struct timer_list rxtick; /* Rx rate control timer */ 568 uint32_t rxticks_per_sec; /* Rx rate control update frequency */ 569 uint32_t rxtick_jiffies; /* Time between updates (in jiffies) */ 570 uint32_t rxticks; /* Rx rate control debug counter */ 571 uint32_t interrupts; /* Total number of interrupts */ 572 spinlock_t lock; /* Main lock for device */ 573 int dev_no; /* Device number (from BDE) */ 574 int cpu_no; /* Cpu number. 1 for iHost(AXI),0 for others */ 575 int dcb_type; /* DCB type */ 576 int dcb_wsize; /* DCB size (in 32-bit words) */ 577 int pkt_hdr_size; /* Packet header size */ 578 uint32_t ftmh_lb_key_ext_size; /* FTMH LB-Key Extension existence/size */ 579 uint32_t ftmh_stacking_ext_size; /* FTMH Stacking extension existence/size */ 580 uint32_t pph_base_size; /* Size of PPH base */ 581 uint32_t pph_lif_ext_size[8]; /* Size of PPH Lif extension header */ 582 uint32_t udh_length_type[4]; /* Size of UDH header per type */ 583 uint32_t udh_size; /* Size of UDH header on legacy devices */ 584 uint32_t oamp_punt; /* OAMP port if nonzero */ 585 uint8_t no_skip_udh_check; /* Indicates UDH won't be skipped */ 586 uint8_t system_headers_mode; /* Indicates system header mode */ 587 uint8_t udh_enable; /* Indicates UDH existence */ 588 int rx_chans; /* Number of Rx channels */ 589 uint32_t dma_hi; /* DMA higher address */ 590 uint32_t cmic_type; /* CMIC type (CMICe or CMICm) */ 591 uint32_t irq_mask; /* Active IRQs for DMA control */ 592 uint32_t napi_poll_mode; /* NAPI is in polling mode */ 593 uint32_t napi_not_done; /* NAPI poll did not process all packets */ 594 uint32_t napi_poll_again; /* Used if DCB chain is restarted */ 595 uint32_t tx_yield; /* Tx schedule for Continuous DMA and Non-NAPI mode */ 596 void *dcb_mem; /* Logical pointer to DCB memory */ 597 uint64_t dcb_dma; /* Physical bus address for DCB memory */ 598 int dcb_mem_size; /* Total size of allocated DCB memory */ 599 uint32_t dma_events; /* DMA events pending for BCM API */ 600 uint32_t rcpu_sig; /* RCPU signature */ 601 uint64_t halt_addr[NUM_DMA_CHAN]; /* DMA halt address */ 602 uint32_t cdma_channels; /* Active channels for Continuous DMA mode */ 603 uint32_t poll_channels; /* Channels for polling */ 604 uint32_t inst_id; /* Instance id of this device */ 605 int evt_idx; /* Event queue index for this device*/ 606 int basedev_suspended; /* Base device suspended */ 607 struct sk_buff_head tx_ptp_queue; /* Tx PTP skb queue */ 608 struct work_struct tx_ptp_work; /* Tx PTP work */ 609 struct { 610 bkn_desc_info_t desc[MAX_TX_DCBS+1]; 611 int free; /* Number of free Tx DCBs */ 612 int cur; /* Index of current Tx DCB */ 613 int dirty; /* Index of next Tx DCB to complete */ 614 int api_active; /* BCM Tx API is in progress */ 615 int suspends; /* Calls to netif_stop_queue (debug only) */ 616 struct list_head api_dcb_list; /* Tx DCB chains from BCM Tx API */ 617 bkn_dcb_chain_t *api_dcb_chain; /* Current Tx DCB chain */ 618 bkn_dcb_chain_t *api_dcb_chain_end; /* Tx DCB chain end */ 619 uint32_t pkts; /* Tx packet counter */ 620 uint32_t pkts_d_no_skb; /* Tx drop - skb allocation failed */ 621 uint32_t pkts_d_rcpu_encap; /* Tx drop - bad RCPU encapsulation */ 622 uint32_t pkts_d_rcpu_sig; /* Tx drop - bad RCPU signature */ 623 uint32_t pkts_d_rcpu_meta; /* Tx drop - bad RCPU meta data */ 624 uint32_t pkts_d_pad_fail; /* Tx drop - pad to minimum size failed */ 625 uint32_t pkts_d_dma_resrc; /* Tx drop - no DMA resources */ 626 uint32_t pkts_d_callback; /* Tx drop - consumed by call-back */ 627 uint32_t pkts_d_no_link; /* Tx drop - software link down */ 628 uint32_t pkts_d_over_limit; /* Tx drop - length is out of range */ 629 } tx; 630 struct { 631 bkn_desc_info_t desc[MAX_RX_DCBS+1]; 632 int free; /* Number of free Rx DCBs */ 633 int cur; /* Index of current Rx DCB */ 634 int dirty; /* Index of next Rx DCB to complete */ 635 int running; /* Rx DMA is active */ 636 int api_active; /* BCM Rx API is active */ 637 int chain_complete; /* All DCBs in chain processed */ 638 int sync_err; /* Chain done with incomplete DCBs (debug) */ 639 int sync_retry; /* Total retry times for sync error (debug) */ 640 int sync_maxloop; /* Max loop times once in recovering sync (debug) */ 641 int use_rx_skb; /* Use SKBs for DMA */ 642 uint32_t rate_max; /* Rx rate in packets/sec */ 643 uint32_t burst_max; /* Rx burst size in number of packets */ 644 uint32_t tokens; /* Tokens for Rx rate control */ 645 uint32_t rate; /* Current packet rate */ 646 unsigned long tok_jif; /* Jiffies at last token update */ 647 unsigned long rate_jif; /* Jiffies at last rate update */ 648 struct list_head api_dcb_list; /* Rx DCB chains from BCM Rx API */ 649 bkn_dcb_chain_t *api_dcb_chain; /* Current Rx DCB chain */ 650 bkn_dcb_chain_t *api_dcb_chain_end; /* Rx DCB chain end */ 651 uint32_t pkts; /* Rx packet counter */ 652 uint32_t pkts_ref; /* Rx packet count for rate calculation */ 653 uint32_t pkts_f_api; /* Rx packets filtered to API */ 654 uint32_t pkts_f_netif; /* Rx packets filtered to net interface */ 655 uint32_t pkts_m_api; /* Rx packets mirrored to API */ 656 uint32_t pkts_m_netif; /* Rx packets mirrored to net interface */ 657 uint32_t pkts_d_no_skb; /* Rx drop - skb allocation failed */ 658 uint32_t pkts_d_no_match; /* Rx drop - no matching filters */ 659 uint32_t pkts_d_unkn_netif; /* Rx drop - unknown net interface ID */ 660 uint32_t pkts_d_unkn_dest; /* Rx drop - unknown destination type */ 661 uint32_t pkts_d_callback; /* Rx drop - consumed by call-back */ 662 uint32_t pkts_d_no_link; /* Rx drop - software link down */ 663 uint32_t pkts_d_no_api_buf; /* Rx drop - no API buffers */ 664 } rx[NUM_RX_CHAN]; 665 } bkn_switch_info_t; 666 667 /* 0x1 - Jericho 2 mode */ 668 #define BKN_DNX_JR2_MODE 1 669 /* PTCH_2 */ 670 #define BKN_DNX_PTCH_2_SIZE 2 671 /* ITMH */ 672 #define BKN_DNX_ITMH_SIZE 5 673 /* Modlue Header */ 674 #define BKN_DNX_MODULE_HEADER_SIZE 20 675 /* FTMH */ 676 #define BKN_DNX_FTMH_SRC_SYS_PORT_AGGREGATE_MSB 17 677 #define BKN_DNX_FTMH_SRC_SYS_PORT_AGGREGATE_NOF_BITS 16 678 #define BKN_DNX_FTMH_PP_DSP_MSB 33 679 #define BKN_DNX_FTMH_PP_DSP_NOF_BITS 8 680 #define BKN_DNX_FTMH_ACTION_TYPE_MSB 43 681 #define BKN_DNX_FTMH_ACTION_TYPE_NOF_BITS 2 682 #define BKN_DNX_FTMH_PPH_TYPE_IS_TSH_EN_MSB 73 683 #define BKN_DNX_FTMH_PPH_TYPE_IS_TSH_EN_NOF_BITS 1 684 #define BKN_DNX_FTMH_PPH_TYPE_IS_PPH_EN_MSB 74 685 #define BKN_DNX_FTMH_PPH_TYPE_IS_PPH_EN_NOF_BITS 1 686 #define BKN_DNX_FTMH_TM_DST_EXT_PRESENT_MSB 75 687 #define BKN_DNX_FTMH_TM_DST_EXT_PRESENT_NOF_BITS 1 688 #define BKN_DNX_FTMH_APP_SPECIFIC_EXT_SIZE_MSB 76 689 #define BKN_DNX_FTMH_APP_SPECIFIC_EXT_SIZE_NOF_BITS 1 690 #define BKN_DNX_FTMH_FLOW_ID_EXT_SIZE_MSB 77 691 #define BKN_DNX_FTMH_FLOW_ID_EXT_SIZE_NOF_BITS 1 692 #define BKN_DNX_FTMH_BIER_BFR_EXT_SIZE_MSB 78 693 #define BKN_DNX_FTMH_BIER_BFR_EXT_SIZE_NOF_BITS 1 694 /* Fix Length for FTMH and Extension headers */ 695 #define BKN_DNX_FTMH_BASE_SIZE 10 696 #define BKN_DNX_FTMH_BIER_BFR_EXT_SIZE 2 697 #define BKN_DNX_FTMH_TM_DST_EXT_SIZE 3 698 #define BKN_DNX_FTMH_FLOW_ID_EXT_SIZE 3 699 #define BKN_DNX_FTMH_APP_SPECIFIC_EXT_SIZE 6 700 /* TSH */ 701 #define BKN_DNX_TSH_SIZE 4 702 /* PPH */ 703 #define BKN_DNX_INTERNAL_BASE_TYPE_9 9 704 #define BKN_DNX_INTERNAL_BASE_TYPE_10 10 705 #define BKN_DNX_INTERNAL_BASE_TYPE_12 12 706 #define BKN_DNX_INTERNAL_9_FORWARD_DOMAIN_MSB 5 707 #define BKN_DNX_INTERNAL_9_FORWARD_DOMAIN_NOF_BITS 16 708 #define BKN_DNX_INTERNAL_9_LEARN_EXT_PRESENT_MSB 53 709 #define BKN_DNX_INTERNAL_9_LEARN_EXT_PRESENT_NOF_BITS 1 710 #define BKN_DNX_INTERNAL_9_FHEI_SIZE_MSB 54 711 #define BKN_DNX_INTERNAL_9_FHEI_SIZE_NOF_BITS 2 712 #define BKN_DNX_INTERNAL_9_LIF_EXT_TYPE_MSB 56 713 #define BKN_DNX_INTERNAL_9_LIF_EXT_TYPE_NOF_BITS 3 714 #define BKN_DNX_INTERNAL_10_FORWARD_DOMAIN_MSB 9 715 #define BKN_DNX_INTERNAL_10_FORWARD_DOMAIN_NOF_BITS 16 716 #define BKN_DNX_INTERNAL_10_LEARN_EXT_PRESENT_MSB 61 717 #define BKN_DNX_INTERNAL_10_LEARN_EXT_PRESENT_NOF_BITS 1 718 #define BKN_DNX_INTERNAL_10_FHEI_SIZE_MSB 62 719 #define BKN_DNX_INTERNAL_10_FHEI_SIZE_NOF_BITS 2 720 #define BKN_DNX_INTERNAL_10_LIF_EXT_TYPE_MSB 64 721 #define BKN_DNX_INTERNAL_10_LIF_EXT_TYPE_NOF_BITS 3 722 #define BKN_DNX_INTERNAL_12_FORWARD_DOMAIN_MSB 21 723 #define BKN_DNX_INTERNAL_12_FORWARD_DOMAIN_NOF_BITS 18 724 #define BKN_DNX_INTERNAL_12_LEARN_EXT_PRESENT_MSB 77 725 #define BKN_DNX_INTERNAL_12_LEARN_EXT_PRESENT_NOF_BITS 1 726 #define BKN_DNX_INTERNAL_12_FHEI_SIZE_MSB 78 727 #define BKN_DNX_INTERNAL_12_FHEI_SIZE_NOF_BITS 2 728 #define BKN_DNX_INTERNAL_12_LIF_EXT_TYPE_MSB 80 729 #define BKN_DNX_INTERNAL_12_LIF_EXT_TYPE_NOF_BITS 3 730 /* PPH.FHEI_TYPE */ 731 #define BKN_DNX_INTERNAL_FHEI_TYPE_SZ0 1 732 #define BKN_DNX_INTERNAL_FHEI_TYPE_SZ1 2 733 #define BKN_DNX_INTERNAL_FHEI_TYPE_SZ2 3 734 /* FHEI */ 735 #define BKN_DNX_INTERNAL_FHEI_SZ0_SIZE 3 736 #define BKN_DNX_INTERNAL_FHEI_SZ1_SIZE 5 737 #define BKN_DNX_INTERNAL_FHEI_SZ2_SIZE 8 738 #define BKN_DNX_INTERNAL_FHEI_TRAP_5B_QUALIFIER_MSB 0 739 #define BKN_DNX_INTERNAL_FHEI_TRAP_5B_QUALIFIER_NOF_BITS 27 740 #define BKN_DNX_INTERNAL_FHEI_TRAP_5B_CODE_MSB 27 741 #define BKN_DNX_INTERNAL_FHEI_TRAP_5B_CODE_NOF_BITS 9 742 #define BKN_DNX_INTERNAL_FHEI_TRAP_5B_TYPE_MSB 36 743 #define BKN_DNX_INTERNAL_FHEI_TRAP_5B_TYPE_NOF_BITS 4 744 /* PPH Extension */ 745 #define BKN_DNX_INTERNAL_LEARN_EXT_SIZE 19 746 /* UDH */ 747 #define BKN_DNX_UDH_DATA_TYPE_0_MSB 0 748 #define BKN_DNX_UDH_DATA_TYPE_0_NOF_BITS 2 749 #define BKN_DNX_UDH_DATA_TYPE_1_MSB 2 750 #define BKN_DNX_UDH_DATA_TYPE_1_NOF_BITS 2 751 #define BKN_DNX_UDH_DATA_TYPE_2_MSB 4 752 #define BKN_DNX_UDH_DATA_TYPE_2_NOF_BITS 2 753 #define BKN_DNX_UDH_DATA_TYPE_3_MSB 6 754 #define BKN_DNX_UDH_DATA_TYPE_3_NOF_BITS 2 755 #define BKN_DNX_UDH_BASE_SIZE 1 756 757 #define BKN_DPP_HDR_MAX_SIZE 40 758 /* PTCH_2 */ 759 #define BKN_DPP_PTCH_2_SIZE 2 760 /* ITMH */ 761 #define BKN_DPP_ITMH_SIZE 4 762 /* FTMH */ 763 #define BKN_DPP_FTMH_LB_EXT_EN 0x1 764 #define BKN_DPP_FTMH_STACKING_EXT_EN 0x2 765 #define BKN_DPP_FTMH_SIZE_BYTE 9 766 #define BKN_DPP_FTMH_LB_EXT_SIZE_BYTE 1 767 #define BKN_DPP_FTMH_STACKING_SIZE_BYTE 2 768 #define BKN_DPP_FTMH_DEST_EXT_SIZE_BYTE 2 769 #define BKN_DPP_FTMH_LB_EXT_SIZE_BYTE 1 770 #define BKN_DPP_FTMH_PKT_SIZE_MSB 0 771 #define BKN_DPP_FTMH_PKT_SIZE_NOF_BITS 14 772 #define BKN_DPP_FTMH_TC_MSB 14 773 #define BKN_DPP_FTMH_TC_NOF_BITS 3 774 #define BKN_DPP_FTMH_SRC_SYS_PORT_MSB 17 775 #define BKN_DPP_FTMH_SRC_SYS_PORT_NOF_BITS 16 776 #define BKN_DPP_FTMH_EXT_DSP_EXIST_MSB 68 777 #define BKN_DPP_FTMH_EXT_DSP_EXIST_NOF_BITS 1 778 #define BKN_DPP_FTMH_EXT_MSB 45 779 #define BKN_DPP_FTMH_EXT_NOF_BITS 2 780 #define BKN_DPP_FTMH_FIRST_EXT_MSB 72 781 #define BKN_DPP_FTMH_ACTION_TYPE_MSB 43 782 #define BKN_DPP_FTMH_ACTION_TYPE_NOF_BITS 2 783 #define BKN_DPP_FTMH_PPH_TYPE_MSB 45 784 #define BKN_DPP_FTMH_PPH_TYPE_NOF_BITS 2 785 786 /* OTSH */ 787 #define BKN_DPP_OTSH_SIZE_BYTE 6 788 #define BKN_DPP_OTSH_TYPE_MSB 0 789 #define BKN_DPP_OTSH_TYPE_NOF_BITS 2 790 #define BKN_DPP_OTSH_OAM_SUB_TYPE_MSB 2 791 #define BKN_DPP_OTSH_OAM_SUB_TYPE_NOF_BITS 3 792 793 #define BKN_DPP_OTSH_TYPE_OAM 0 794 #define BKN_DPP_OTSH_OAM_SUB_TYPE_DM_1588 2 795 #define BKN_DPP_OTSH_OAM_SUB_TYPE_DM_NTP 3 796 797 #define BKN_DPP_OAM_DM_TOD_SIZE_BYTE 4 798 799 /* PPH */ 800 #define BKN_DPP_INTERNAL_SIZE_BYTE 7 801 #define BKN_DPP_INTERNAL_EEI_EXTENSION_PRESENT_MSB 0 802 #define BKN_DPP_INTERNAL_EEI_EXTENSION_PRESENT_NOF_BITS 1 803 #define BKN_DPP_INTERNAL_LEARN_EXENSION_PRESENT_MSB 1 804 #define BKN_DPP_INTERNAL_LEARN_EXENSION_PRESENT_NOF_BITS 1 805 #define BKN_DPP_INTERNAL_FHEI_SIZE_MSB 2 806 #define BKN_DPP_INTERNAL_FHEI_SIZE_NOF_BITS 2 807 #define BKN_DPP_INTERNAL_FORWARD_CODE_MSB 4 808 #define BKN_DPP_INTERNAL_FORWARD_CODE_NOF_BITS 4 809 #define BKN_DPP_INTERNAL_FORWARD_CODE_CPU_TRAP 7 810 #define BKN_DPP_INTERNAL_FORWARDING_HEADER_OFFSET_MSB 8 811 #define BKN_DPP_INTERNAL_FORWARDING_HEADER_OFFSET_NOF_BITS 7 812 #define BKN_DPP_INTERNAL_VSI_MSB 22 813 #define BKN_DPP_INTERNAL_VSI_NOF_BITS 16 814 815 /* FHEI TRAP/SNOOP 3B */ 816 #define BKN_DPP_INTERNAL_FHEI_3B_SIZE_BYTE 3 817 #define BKN_DPP_INTERNAL_FHEI_5B_SIZE_BYTE 5 818 #define BKN_DPP_INTERNAL_FHEI_8B_SIZE_BYTE 8 819 #define BKN_DPP_INTERNAL_FHEI_TRAP_SNOOP_3B_CPU_TRAP_CODE_QUALIFIER_MSB 0 820 #define BKN_DPP_INTERNAL_FHEI_TRAP_SNOOP_3B_CPU_TRAP_CODE_QUALIFIER_NOF_BITS 16 821 #define BKN_DPP_INTERNAL_FHEI_TRAP_SNOOP_3B_CPU_TRAP_CODE_MSB 16 822 #define BKN_DPP_INTERNAL_FHEI_TRAP_SNOOP_3B_CPU_TRAP_CODE_NOF_BITS 8 823 /* PPH extension */ 824 #define BKN_DPP_INTERNAL_EXPLICIT_EDITING_INFOMATION_EXTENSION_SIZE_BYTE 3 825 #define BKN_DPP_INTERNAL_LEARN_EXTENSION_SIZE_BYTE 5 826 827 #define BKN_SAND_SCRATCH_DATA_SIZE 4 828 829 /* ftmh action type. */ 830 typedef enum bkn_dpp_ftmh_action_type_e { 831 BKN_DPP_FTMH_ACTION_TYPE_FORWARD = 0, /* TM action is forward */ 832 BKN_DPP_FTMH_ACTION_TYPE_SNOOP = 1, /* TM action is snoop */ 833 BKN_DPP_FTMH_ACTION_TYPE_INBOUND_MIRROR = 2, /* TM action is inbound mirror. */ 834 BKN_DPP_FTMH_ACTION_TYPE_OUTBOUND_MIRROR = 3 /* TM action is outbound mirror. */ 835 }bkn_dpp_ftmh_action_type_t; 836 837 /* ftmh dest extension. */ 838 typedef struct bkn_dpp_ftmh_dest_extension_s { 839 uint8_t valid; /* Set if the extension is present */ 840 uint32_t dst_sys_port; /* Destination System Port */ 841 } bkn_dpp_ftmh_dest_extension_t; 842 843 /* dnx packet */ 844 typedef struct bkn_dune_system_header_info_s { 845 uint32_t system_header_size; 846 struct { 847 uint32_t action_type; /* Indicates if the copy is one of the Forward Snoop or Mirror packet copies */ 848 uint32_t source_sys_port_aggregate; /* Source System port*/ 849 } ftmh; 850 struct { 851 uint32_t forward_domain; 852 uint32_t trap_qualifier; 853 uint32_t trap_id; 854 } internal; 855 } bkn_dune_system_header_info_t; 856 857 #define PREV_IDX(_cur, _max) (((_cur) == 0) ? (_max) - 1 : (_cur) - 1) 858 859 #if defined(CMIC_SOFT_BYTE_SWAP) 860 861 #define CMIC_SWAP32(_x) ((((_x) & 0xff000000) >> 24) \ 862 | (((_x) & 0x00ff0000) >> 8) \ 863 | (((_x) & 0x0000ff00) << 8) \ 864 | (((_x) & 0x000000ff) << 24)) 865 866 #define DEV_READ32(_d, _a, _p) \ 867 do { \ 868 uint32_t _data; \ 869 _data = (((volatile uint32_t *)(_d)->base_addr)[(_a)/4]); \ 870 *(_p) = CMIC_SWAP32(_data); \ 871 } while(0) 872 873 #define DEV_WRITE32(_d, _a, _v) \ 874 do { \ 875 uint32_t _data = CMIC_SWAP32(_v); \ 876 ((volatile uint32_t *)(_d)->base_addr)[(_a)/4] = (_data); \ 877 } while(0) 878 879 #else 880 881 #define DEV_READ32(_d, _a, _p) \ 882 do { \ 883 *(_p) = (((volatile uint32_t *)(_d)->base_addr)[(_a)/4]); \ 884 } while(0) 885 886 #define DEV_WRITE32(_d, _a, _v) \ 887 do { \ 888 ((volatile uint32_t *)(_d)->base_addr)[(_a)/4] = (_v); \ 889 } while(0) 890 891 #endif /* defined(CMIC_SOFT_BYTE_SWAP) */ 892 893 #define MEMORY_BARRIER smp_mb() 894 895 /* Default random MAC address has Broadcom OUI with local admin bit set */ 896 static u8 bkn_dev_mac[6] = { 0x02, 0x10, 0x18, 0x00, 0x00, 0x00 }; 897 898 static u8 bkn_rcpu_dmac[6]; 899 static u8 bkn_rcpu_smac[6]; 900 901 /* Driver Proc Entry root */ 902 static struct proc_dir_entry *bkn_proc_root = NULL; 903 904 typedef struct bkn_priv_s { 905 struct list_head list; 906 struct net_device_stats stats; 907 struct net_device *dev; 908 bkn_switch_info_t *sinfo; 909 int id; 910 int type; 911 int port; 912 int qnum; 913 uint32_t vlan; 914 uint32_t flags; 915 uint32_t cb_user_data; 916 uint8_t system_headers[27]; 917 uint32_t system_headers_size; 918 int tx_hwts; /* HW timestamp for Tx */ 919 int rx_hwts; /* HW timestamp for Rx */ 920 int phys_port; 921 } bkn_priv_t; 922 923 typedef struct bkn_filter_s { 924 struct list_head list; 925 int dev_no; 926 unsigned long hits; 927 kcom_filter_t kf; 928 } bkn_filter_t; 929 930 931 /* 932 * Multiple instance support in KNET 933 */ 934 static int _bkn_multi_inst = 0; 935 typedef struct { 936 wait_queue_head_t evt_wq; 937 int evt_wq_put; 938 int evt_wq_get; 939 uint32_t inst_id; 940 } bkn_evt_resource_t; 941 942 static bkn_evt_resource_t _bkn_evt[LINUX_BDE_MAX_DEVICES]; 943 944 static int bkn_knet_dev_init(int d); 945 static int bkn_knet_dev_reinit(int d); 946 947 /* IOCTL debug counters */ 948 static int ioctl_cmd; 949 static int ioctl_evt; 950 951 /* Switch devices */ 952 LIST_HEAD(_sinfo_list); 953 954 /* Reallocation chunk size for netif array */ 955 #define NDEVS_CHUNK 64 956 957 /* User call-backs */ 958 static knet_skb_cb_f knet_rx_cb = NULL; 959 static knet_skb_cb_f knet_tx_cb = NULL; 960 static knet_filter_cb_f knet_filter_cb = NULL; 961 static knet_hw_tstamp_enable_cb_f knet_hw_tstamp_enable_cb = NULL; 962 static knet_hw_tstamp_enable_cb_f knet_hw_tstamp_disable_cb = NULL; 963 static knet_hw_tstamp_tx_time_get_cb_f knet_hw_tstamp_tx_time_get_cb = NULL; 964 static knet_hw_tstamp_tx_meta_get_cb_f knet_hw_tstamp_tx_meta_get_cb = NULL; 965 static knet_hw_tstamp_ptp_clock_index_cb_f knet_hw_tstamp_ptp_clock_index_cb = NULL; 966 static knet_hw_tstamp_rx_time_upscale_cb_f knet_hw_tstamp_rx_time_upscale_cb = NULL; 967 static knet_hw_tstamp_ioctl_cmd_cb_f knet_hw_tstamp_ioctl_cmd_cb = NULL; 968 969 /* 970 * Thread management 971 */ 972 973 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,10)) 974 static void 975 bkn_sleep(int clicks) 976 { 977 wait_queue_head_t wq; 978 979 init_waitqueue_head(&wq); 980 sleep_on_timeout(&wq, clicks); 981 } 982 #else 983 static void 984 bkn_sleep(int clicks) 985 { 986 wait_queue_head_t wq; 987 988 init_waitqueue_head(&wq); 989 wait_event_timeout(wq, 0, clicks); 990 } 991 #endif 992 993 /* 994 * On XGS devices bit 15 fo the Transferred Bytes field in 995 * the DCBs is used to indicate that kernel processing is 996 * complete. Using this bit reduces the theoretically maximum 997 * supported packet size from 64K to 32K, but this is still 998 * adequate for 16K jumbo packets. 999 */ 1000 #define SOC_DCB_KNET_DONE 0x8000 1001 #define SOC_DCB_KNET_COUNT_MASK 0x7fff 1002 #define SOC_DCB_META_OFFSET 2 1003 1004 /* Default channel configuration */ 1005 #define XGS_DMA_TX_CHAN 0 1006 #define XGS_DMA_RX_CHAN 1 1007 1008 /* CMIC registers */ 1009 #define CMIC_DMA_CTRLr 0x00000100 1010 #define CMIC_DMA_STATr 0x00000104 1011 #define CMIC_DMA_DESC0r 0x00000110 1012 #define CMIC_IRQ_STATr 0x00000144 1013 #define CMIC_IRQ_MASKr 0x00000148 1014 #define CMIC_DEV_REV_IDr 0x00000178 1015 1016 /* CMIC interrupts reserved for kernel handler */ 1017 #define CMIC_TXRX_IRQ_MASK 0x7f80 1018 1019 /* CMICm registers */ 1020 #define CMICM_CMC_BASE 0x00031000 1021 #define CMICM_DMA_CTRLr (CMICM_CMC_BASE + 0x00000140) 1022 #define CMICM_DMA_STATr (CMICM_CMC_BASE + 0x00000150) 1023 #define CMICM_DMA_STAT_CLRr (CMICM_CMC_BASE + 0x000001a4) 1024 #define CMICM_DMA_DESC0r (CMICM_CMC_BASE + 0x00000158) 1025 #define CMICM_DMA_HALT_ADDRr (CMICM_CMC_BASE + 0x00000120) 1026 #define CMICM_IRQ_STATr (CMICM_CMC_BASE + 0x00000400) 1027 #define CMICM_IRQ_PCI_MASKr (CMICM_CMC_BASE + 0x00000414) 1028 #define CMICM_IRQ_UC0_MASKr (CMICM_CMC_BASE + 0x00000428) 1029 #define CMICM_DEV_REV_IDr 0x00010224 1030 1031 /* CMICm interrupts reserved for kernel handler */ 1032 #define CMICM_TXRX_IRQ_MASK 0xff00 1033 1034 /* CMICd increased interrupts reserved for kernel handler */ 1035 #define CMICD_CTRLD_IRQ_MASK 0x78000000 1036 1037 /* CMICx registers */ 1038 #define CMICX_CMC_BASE 0x00000000 1039 #define CMICX_DMA_CTRLr (CMICX_CMC_BASE + 0x00002100) 1040 #define CMICX_DMA_STATr (CMICX_CMC_BASE + 0x00002114) 1041 #define CMICX_DMA_DESC_HIr (CMICX_CMC_BASE + 0x00002108) 1042 #define CMICX_DMA_DESC_LOr (CMICX_CMC_BASE + 0x00002104) 1043 #define CMICX_DMA_HALT_HIr (CMICX_CMC_BASE + 0x00002110) 1044 #define CMICX_DMA_HALT_LOr (CMICX_CMC_BASE + 0x0000210c) 1045 #define CMICX_IRQ_STATr (CMICX_CMC_BASE + 0x0000106c) 1046 #define CMICX_IRQ_STAT_CLRr (CMICX_CMC_BASE + 0x00001074) 1047 #define CMICX_IRQ_ENABr 0x18013100 1048 #define IHOST_GIC_GIC400_GICD_ISENABLERN_5r 0x10781114 1049 #define IHOST_GIC_GIC400_GICD_ICENABLERN_5r 0x10781194 1050 1051 /* CMICx interrupts reserved for kernel handler */ 1052 #define CMICX_TXRX_IRQ_MASK 0xffffffff 1053 1054 #define DEV_IS_CMICX(_sinfo) ((_sinfo)->cmic_type == 'x') 1055 #define DEV_IS_CMICM(_sinfo) ((_sinfo)->cmic_type == 'm') 1056 #define CDMA_CH(_d, _ch) ((_d)->cdma_channels & (1 << (_ch))) 1057 1058 /* 1059 * DMA_STAT: control bits 1060 * 1061 * xxx_SET and xxx_CLR can be WRITTEN to CMIC_DMA_STAT 1062 * xxx_TST can be masked against values read from CMIC_DMA_STAT. 1063 * Argument required: 0 <= ch <= 3 1064 */ 1065 #define DS_DMA_ACTIVE(ch) (0x00040000 << (ch)) 1066 #define DS_DMA_EN_SET(ch) (0x80|(ch)) 1067 #define DS_DMA_EN_CLR(ch) (0x00|(ch)) 1068 #define DS_DMA_EN_TST(ch) (0x00000001 << (ch)) 1069 1070 #define DS_CHAIN_DONE_SET(ch) (0x80|(4+(ch))) 1071 #define DS_CHAIN_DONE_CLR(ch) (0x00|(4+(ch))) 1072 #define DS_CHAIN_DONE_TST(ch) (0x00000010 << (ch)) 1073 1074 #define DS_DESC_DONE_SET(ch) (0x80|(8+(ch))) 1075 #define DS_DESC_DONE_CLR(ch) (0x00|(8+(ch))) 1076 #define DS_DESC_DONE_TST(ch) (0x00000100 << (ch)) 1077 1078 #define DC_ABORT_DMA(ch) (0x04 << (8 * (ch))) 1079 1080 #define DS_CMC_DESCRD_CMPLT_CLR(ch) (0x00000001 << (ch)) 1081 #define DS_CMC_CTRLD_INT_CLR(ch) (0x00000100 << (ch)) 1082 #define DS_CMC_DMA_ACTIVE(ch) (0x00000100 << (ch)) 1083 1084 #define CMICX_DS_CMC_DESC_DONE(ch) (0x00000001 << ((ch) * 4)) 1085 #define CMICX_DS_CMC_CHAIN_DONE(ch) (0x00000002 << ((ch) * 4)) 1086 #define CMICX_DS_CMC_CTRLD_INT(ch) (0x00000008 << ((ch) * 4)) 1087 #define CMICX_DS_CMC_DMA_CHAIN_DONE (0x00000001) 1088 #define CMICX_DS_CMC_DMA_ACTIVE (0x00000002) 1089 1090 #define DMA_TO_BUS_HI(dma) ((dma) | sinfo->dma_hi) 1091 #define BUS_TO_DMA_HI(bus) ((bus) & ~sinfo->dma_hi) 1092 1093 /* 1094 * DMA_CTRL: control bits 1095 */ 1096 #define DC_CMC_DIRECTION (0x00000001) 1097 #define DC_CMC_ENABLE (0x00000002) 1098 #define DC_CMC_ABORT (0x00000004) 1099 #define DC_CMC_CTRLD_INT (0x00000100) 1100 #define DC_CMC_CONTINUOUS (0x00000200) 1101 1102 #define CMICX_DC_CMC_DIRECTION (0x00000001) 1103 #define CMICX_DC_CMC_ENABLE (0x00000002) 1104 #define CMICX_DC_CMC_ABORT (0x00000004) 1105 #define CMICX_DC_CMC_CTRLD_INT (0x00000080) 1106 #define CMICX_DC_CMC_CONTINUOUS (0x00000100) 1107 1108 /* 1109 * Per-channel operations. 1110 * These are the basis for the TX/RX functions 1111 */ 1112 1113 static inline void 1114 xgs_dma_chain_clear(bkn_switch_info_t *sinfo, int chan) 1115 { 1116 DBG_IRQ(("Clear chain on device %d chan %d\n", 1117 sinfo->dev_no, chan)); 1118 DEV_WRITE32(sinfo, CMIC_DMA_STATr, DS_DMA_EN_CLR(chan)); 1119 DEV_WRITE32(sinfo, CMIC_DMA_STATr, DS_CHAIN_DONE_CLR(chan)); 1120 1121 MEMORY_BARRIER; 1122 } 1123 1124 static inline void 1125 xgs_dma_desc_clear(bkn_switch_info_t *sinfo, int chan) 1126 { 1127 uint32_t val; 1128 1129 DBG_IRQ(("Clear desc on device %d chan %d\n", 1130 sinfo->dev_no, chan)); 1131 DEV_WRITE32(sinfo, CMIC_DMA_STATr, DS_DESC_DONE_CLR(chan)); 1132 1133 MEMORY_BARRIER; 1134 1135 /* Flush write buffer */ 1136 DEV_READ32(sinfo, CMIC_DMA_STATr, &val); 1137 1138 MEMORY_BARRIER; 1139 } 1140 1141 static int 1142 xgs_dma_chan_clear(bkn_switch_info_t *sinfo, int chan) 1143 { 1144 xgs_dma_chain_clear(sinfo, chan); 1145 xgs_dma_desc_clear(sinfo, chan); 1146 1147 return 0; 1148 } 1149 1150 static int 1151 xgs_dma_chan_init(bkn_switch_info_t *sinfo, int chan, int dir) 1152 { 1153 uint32_t cdc; 1154 1155 DEV_READ32(sinfo, CMIC_DMA_CTRLr, &cdc); 1156 1157 cdc &= ~(0x9 << (8 * chan)); 1158 if (dir) { 1159 cdc |= 0x1 << (8 * chan); 1160 } else { 1161 cdc |= 0x8 << (8 * chan); 1162 } 1163 1164 DEV_WRITE32(sinfo, CMIC_DMA_CTRLr, cdc); 1165 1166 return 0; 1167 } 1168 1169 static int 1170 xgs_dma_chan_start(bkn_switch_info_t *sinfo, int chan, uint64_t dcb) 1171 { 1172 /* Write the DCB address to the DESC address for this channel */ 1173 DEV_WRITE32(sinfo, CMIC_DMA_DESC0r + 4 * chan, dcb); 1174 1175 MEMORY_BARRIER; 1176 1177 /* Kick it off */ 1178 DEV_WRITE32(sinfo, CMIC_DMA_STATr, DS_DMA_EN_SET(chan)); 1179 1180 MEMORY_BARRIER; 1181 1182 return 0; 1183 } 1184 1185 static int 1186 xgs_dma_chan_abort(bkn_switch_info_t *sinfo, int chan, int polls) 1187 { 1188 uint32_t ctrl, dma_stat; 1189 int p; 1190 1191 /* Clear enable */ 1192 DEV_WRITE32(sinfo, CMIC_DMA_STATr, DS_DMA_EN_CLR(chan)); 1193 1194 MEMORY_BARRIER; 1195 1196 /* Abort the channel */ 1197 DEV_READ32(sinfo, CMIC_DMA_CTRLr, &ctrl); 1198 DEV_WRITE32(sinfo, CMIC_DMA_CTRLr, ctrl | DC_ABORT_DMA(chan)); 1199 1200 MEMORY_BARRIER; 1201 1202 /* Poll for abort completion */ 1203 for (p = 0; p < polls; p++) { 1204 DEV_READ32(sinfo, CMIC_DMA_STATr, &dma_stat); 1205 if (!(dma_stat & DS_DMA_ACTIVE(chan))) { 1206 /* Restore previous control value */ 1207 DEV_WRITE32(sinfo, CMIC_DMA_CTRLr, ctrl); 1208 1209 MEMORY_BARRIER; 1210 1211 /* Clear up channel */ 1212 xgs_dma_chan_clear(sinfo, chan); 1213 1214 return polls; 1215 } 1216 } 1217 DBG_WARN(("DMA channel %d abort failed\n", chan)); 1218 1219 return -1; 1220 } 1221 1222 static inline void 1223 xgs_irq_mask_set(bkn_switch_info_t *sinfo, uint32_t mask) 1224 { 1225 if (sinfo->napi_poll_mode) { 1226 mask = 0; 1227 } 1228 lkbde_irq_mask_set(sinfo->dev_no | LKBDE_ISR2_DEV, CMIC_IRQ_MASKr, 1229 mask, CMIC_TXRX_IRQ_MASK); 1230 } 1231 1232 static inline void 1233 xgs_irq_mask_enable(bkn_switch_info_t *sinfo, int chan, int update_hw) 1234 { 1235 uint32_t mask; 1236 1237 if (chan == XGS_DMA_TX_CHAN) { 1238 mask = 0x100; 1239 } else { 1240 mask = 0x180 << (2 * chan); 1241 } 1242 1243 sinfo->irq_mask |= mask; 1244 1245 if (update_hw) { 1246 xgs_irq_mask_set(sinfo, sinfo->irq_mask); 1247 } 1248 } 1249 1250 static inline void 1251 xgs_irq_mask_disable(bkn_switch_info_t *sinfo, int chan, int update_hw) 1252 { 1253 uint32_t mask; 1254 1255 if (chan == XGS_DMA_TX_CHAN) { 1256 mask = 0x100; 1257 } else { 1258 mask = 0x180 << (2 * chan); 1259 } 1260 1261 sinfo->irq_mask &= ~mask; 1262 1263 if (update_hw) { 1264 xgs_irq_mask_set(sinfo, sinfo->irq_mask); 1265 } 1266 } 1267 1268 static inline void 1269 xgsm_dma_chain_clear(bkn_switch_info_t *sinfo, int chan) 1270 { 1271 uint32_t cdc; 1272 1273 /* Disabing DMA clears chain done */ 1274 DEV_READ32(sinfo, CMICM_DMA_CTRLr + 4 * chan, &cdc); 1275 cdc &= ~(DC_CMC_ENABLE | DC_CMC_ABORT); 1276 DEV_WRITE32(sinfo, CMICM_DMA_CTRLr + 4 * chan, cdc); 1277 1278 MEMORY_BARRIER; 1279 } 1280 1281 static inline void 1282 xgsm_dma_desc_clear(bkn_switch_info_t *sinfo, int chan) 1283 { 1284 uint32_t val; 1285 1286 val = DS_CMC_DESCRD_CMPLT_CLR(chan); 1287 if (CDMA_CH(sinfo, chan)) { 1288 val |= DS_CMC_CTRLD_INT_CLR(chan); 1289 } 1290 DEV_WRITE32(sinfo, CMICM_DMA_STAT_CLRr, val); 1291 1292 MEMORY_BARRIER; 1293 1294 /* Flush write buffer */ 1295 DEV_READ32(sinfo, CMICM_DMA_STAT_CLRr, &val); 1296 1297 MEMORY_BARRIER; 1298 } 1299 1300 static int 1301 xgsm_dma_chan_clear(bkn_switch_info_t *sinfo, int chan) 1302 { 1303 xgsm_dma_chain_clear(sinfo, chan); 1304 xgsm_dma_desc_clear(sinfo, chan); 1305 1306 return 0; 1307 } 1308 1309 static inline void 1310 xgsm_cdma_halt_set(bkn_switch_info_t *sinfo, int chan) 1311 { 1312 DEV_WRITE32(sinfo, CMICM_DMA_HALT_ADDRr + 4 * chan, 1313 sinfo->halt_addr[chan]); 1314 1315 MEMORY_BARRIER; 1316 } 1317 1318 static int 1319 xgsm_dma_chan_init(bkn_switch_info_t *sinfo, int chan, int dir) 1320 { 1321 uint32_t cdc; 1322 1323 DEV_READ32(sinfo, CMICM_DMA_CTRLr + 4 * chan, &cdc); 1324 cdc &= ~DC_CMC_DIRECTION; 1325 if (dir) { 1326 cdc |= DC_CMC_DIRECTION; 1327 } 1328 if (CDMA_CH(sinfo, chan)) { 1329 cdc |= DC_CMC_CONTINUOUS | DC_CMC_CTRLD_INT; 1330 xgsm_cdma_halt_set(sinfo, chan); 1331 } 1332 DEV_WRITE32(sinfo, CMICM_DMA_CTRLr + 4 * chan, cdc); 1333 1334 return 0; 1335 } 1336 1337 static int 1338 xgsm_dma_chan_start(bkn_switch_info_t *sinfo, int chan, uint64_t dcb) 1339 { 1340 uint32_t cdc; 1341 1342 /* Write the DCB address to the DESC address for this channel */ 1343 DEV_WRITE32(sinfo, CMICM_DMA_DESC0r + 4 * chan, dcb); 1344 1345 MEMORY_BARRIER; 1346 1347 /* Kick it off */ 1348 DEV_READ32(sinfo, CMICM_DMA_CTRLr + 4 * chan, &cdc); 1349 cdc |= DC_CMC_ENABLE; 1350 DEV_WRITE32(sinfo, CMICM_DMA_CTRLr + 4 * chan, cdc); 1351 1352 MEMORY_BARRIER; 1353 1354 return 0; 1355 } 1356 1357 static int 1358 xgsm_dma_chan_abort(bkn_switch_info_t *sinfo, int chan, int polls) 1359 { 1360 uint32_t ctrl, dma_stat; 1361 int p; 1362 1363 /* Skip abort sequence if channel is not active */ 1364 DEV_READ32(sinfo, CMICM_DMA_STATr, &dma_stat); 1365 if (!(dma_stat & DS_CMC_DMA_ACTIVE(chan))) { 1366 return 0; 1367 } 1368 1369 /* Abort the channel */ 1370 DEV_READ32(sinfo, CMICM_DMA_CTRLr + 4 * chan, &ctrl); 1371 ctrl |= (DC_CMC_ENABLE | DC_CMC_ABORT); 1372 DEV_WRITE32(sinfo, CMICM_DMA_CTRLr + 4 * chan, ctrl); 1373 1374 MEMORY_BARRIER; 1375 1376 /* Poll for abort completion */ 1377 for (p = 0; p < polls; p++) { 1378 DEV_READ32(sinfo, CMICM_DMA_STATr, &dma_stat); 1379 if (!(dma_stat & DS_CMC_DMA_ACTIVE(chan))) { 1380 /* Clear up channel */ 1381 xgsm_dma_chan_clear(sinfo, chan); 1382 return polls; 1383 } 1384 } 1385 DBG_WARN(("DMA channel %d abort failed\n", chan)); 1386 1387 return -1; 1388 } 1389 1390 static inline void 1391 xgsm_irq_mask_set(bkn_switch_info_t *sinfo, uint32_t mask) 1392 { 1393 uint32_t irq_mask_reg = CMICM_IRQ_PCI_MASKr; 1394 uint32_t ctrld_mask = 0; 1395 1396 if (sinfo->napi_poll_mode) { 1397 mask = 0; 1398 } 1399 if (sinfo->cpu_no == 1) { 1400 irq_mask_reg = CMICM_IRQ_UC0_MASKr; 1401 } 1402 1403 /* Get the Controlled Interrupts mask for Continuous DMA mode */ 1404 ctrld_mask |= (sinfo->cdma_channels << 27) & CMICD_CTRLD_IRQ_MASK; 1405 1406 lkbde_irq_mask_set(sinfo->dev_no | LKBDE_ISR2_DEV, irq_mask_reg, 1407 mask, CMICM_TXRX_IRQ_MASK | ctrld_mask); 1408 } 1409 1410 static inline void 1411 xgsm_irq_mask_enable(bkn_switch_info_t *sinfo, int chan, int update_hw) 1412 { 1413 uint32_t mask; 1414 1415 if (CDMA_CH(sinfo, chan)) { 1416 mask = 0x08000000 << chan; 1417 } else { 1418 if (chan == XGS_DMA_TX_CHAN) { 1419 mask = 0x8000; 1420 } else { 1421 mask = 0xc000 >> (2 * chan); 1422 } 1423 } 1424 1425 sinfo->irq_mask |= mask; 1426 1427 if (update_hw) { 1428 xgsm_irq_mask_set(sinfo, sinfo->irq_mask); 1429 } 1430 } 1431 1432 static inline void 1433 xgsm_irq_mask_disable(bkn_switch_info_t *sinfo, int chan, int update_hw) 1434 { 1435 uint32_t mask; 1436 1437 if (CDMA_CH(sinfo, chan)) { 1438 mask = 0x08000000 << chan; 1439 } else { 1440 if (chan == XGS_DMA_TX_CHAN) { 1441 mask = 0x8000; 1442 } else { 1443 mask = 0xc000 >> (2 * chan); 1444 } 1445 } 1446 1447 sinfo->irq_mask &= ~mask; 1448 1449 if (update_hw) { 1450 xgsm_irq_mask_set(sinfo, sinfo->irq_mask); 1451 } 1452 } 1453 1454 static inline void 1455 xgsx_dma_chain_clear(bkn_switch_info_t *sinfo, int chan) 1456 { 1457 uint32_t ctrl, stat; 1458 1459 /* Disabing DMA clears chain done */ 1460 DEV_READ32(sinfo, CMICX_DMA_CTRLr + 0x80 * chan, &ctrl); 1461 ctrl &= ~(CMICX_DC_CMC_ENABLE | CMICX_DC_CMC_ABORT); 1462 DEV_WRITE32(sinfo, CMICX_DMA_CTRLr + 0x80 * chan, ctrl); 1463 1464 stat = CMICX_DS_CMC_CHAIN_DONE(chan); 1465 DEV_WRITE32(sinfo, CMICX_IRQ_STAT_CLRr, stat); 1466 1467 MEMORY_BARRIER; 1468 1469 /* Flush write buffer */ 1470 DEV_READ32(sinfo, CMICX_IRQ_STAT_CLRr, &stat); 1471 1472 MEMORY_BARRIER; 1473 } 1474 1475 static inline void 1476 xgsx_dma_desc_clear(bkn_switch_info_t *sinfo, int chan) 1477 { 1478 uint32_t stat; 1479 1480 if (CDMA_CH(sinfo, chan)) { 1481 stat = CMICX_DS_CMC_CTRLD_INT(chan); 1482 } else { 1483 stat = CMICX_DS_CMC_DESC_DONE(chan); 1484 } 1485 DEV_WRITE32(sinfo, CMICX_IRQ_STAT_CLRr, stat); 1486 1487 MEMORY_BARRIER; 1488 1489 /* Flush write buffer */ 1490 DEV_READ32(sinfo, CMICX_IRQ_STAT_CLRr, &stat); 1491 1492 MEMORY_BARRIER; 1493 } 1494 1495 static int 1496 xgsx_dma_chan_clear(bkn_switch_info_t *sinfo, int chan) 1497 { 1498 xgsx_dma_chain_clear(sinfo, chan); 1499 xgsx_dma_desc_clear(sinfo, chan); 1500 1501 return 0; 1502 } 1503 1504 static inline void 1505 xgsx_cdma_halt_set(bkn_switch_info_t *sinfo, int chan) 1506 { 1507 DEV_WRITE32(sinfo, CMICX_DMA_HALT_LOr + 0x80 * chan, 1508 sinfo->halt_addr[chan]); 1509 DEV_WRITE32(sinfo, CMICX_DMA_HALT_HIr + 0x80 * chan, 1510 DMA_TO_BUS_HI(sinfo->halt_addr[chan] >> 32)); 1511 1512 MEMORY_BARRIER; 1513 } 1514 1515 static int 1516 xgsx_dma_chan_init(bkn_switch_info_t *sinfo, int chan, int dir) 1517 { 1518 uint32_t ctrl; 1519 1520 DEV_READ32(sinfo, CMICX_DMA_CTRLr + 0x80 * chan, &ctrl); 1521 ctrl &= ~CMICX_DC_CMC_DIRECTION; 1522 if (dir) { 1523 ctrl |= CMICX_DC_CMC_DIRECTION; 1524 } 1525 if (CDMA_CH(sinfo, chan)) { 1526 ctrl |= CMICX_DC_CMC_CONTINUOUS | CMICX_DC_CMC_CTRLD_INT; 1527 xgsx_cdma_halt_set(sinfo, chan); 1528 } 1529 DEV_WRITE32(sinfo, CMICX_DMA_CTRLr + 0x80 * chan, ctrl); 1530 1531 MEMORY_BARRIER; 1532 1533 return 0; 1534 } 1535 1536 static int 1537 xgsx_dma_chan_start(bkn_switch_info_t *sinfo, int chan, uint64_t dcb) 1538 { 1539 uint32_t ctrl; 1540 1541 /* Write the DCB address to the DESC address for this channel */ 1542 DEV_WRITE32(sinfo, CMICX_DMA_DESC_LOr + 0x80 * chan, dcb); 1543 DEV_WRITE32(sinfo, CMICX_DMA_DESC_HIr + 0x80 * chan, DMA_TO_BUS_HI(dcb >> 32)); 1544 1545 MEMORY_BARRIER; 1546 1547 /* Kick it off */ 1548 DEV_READ32(sinfo, CMICX_DMA_CTRLr + 0x80 * chan, &ctrl); 1549 ctrl |= CMICX_DC_CMC_ENABLE; 1550 DEV_WRITE32(sinfo, CMICX_DMA_CTRLr + 0x80 * chan, ctrl); 1551 1552 MEMORY_BARRIER; 1553 1554 return 0; 1555 } 1556 1557 static int 1558 xgsx_dma_chan_abort(bkn_switch_info_t *sinfo, int chan, int polls) 1559 { 1560 uint32_t ctrl, stat; 1561 int p; 1562 1563 /* Skip abort sequence if channel is not active */ 1564 DEV_READ32(sinfo, CMICX_DMA_STATr + 0x80 * chan, &stat); 1565 if (!(stat & CMICX_DS_CMC_DMA_ACTIVE)) { 1566 return 0; 1567 } 1568 1569 /* Abort the channel */ 1570 DEV_READ32(sinfo, CMICX_DMA_CTRLr + 0x80 * chan, &ctrl); 1571 ctrl |= CMICX_DC_CMC_ENABLE | CMICX_DC_CMC_ABORT; 1572 DEV_WRITE32(sinfo, CMICX_DMA_CTRLr + 0x80 * chan, ctrl); 1573 1574 MEMORY_BARRIER; 1575 1576 /* Poll for abort completion */ 1577 for (p = 0; p < polls; p++) { 1578 DEV_READ32(sinfo, CMICX_DMA_STATr + 0x80 * chan, &stat); 1579 if (!(stat & CMICX_DS_CMC_DMA_ACTIVE)) { 1580 /* Clear up channel */ 1581 xgsx_dma_chan_clear(sinfo, chan); 1582 return polls; 1583 } 1584 } 1585 1586 DBG_WARN(("DMA channel %d abort failed\n", chan)); 1587 1588 return -1; 1589 } 1590 1591 static inline void 1592 xgsx_irq_mask_set(bkn_switch_info_t *sinfo, uint32_t mask) 1593 { 1594 uint32_t irq_mask_reg = CMICX_IRQ_ENABr; 1595 uint32_t irq_mask, irq_fmask, disable_mask; 1596 1597 if (sinfo->napi_poll_mode) { 1598 mask = 0; 1599 } 1600 1601 if (sinfo->cpu_no == 1) { 1602 lkbde_irq_mask_get(sinfo->dev_no, &irq_mask, &irq_fmask); 1603 disable_mask = ~mask & (irq_mask & irq_fmask); 1604 if (disable_mask) { 1605 lkbde_irq_mask_set(sinfo->dev_no | LKBDE_ISR2_DEV | LKBDE_IPROC_REG, 1606 IHOST_GIC_GIC400_GICD_ICENABLERN_5r, disable_mask, CMICX_TXRX_IRQ_MASK); 1607 } 1608 irq_mask_reg = IHOST_GIC_GIC400_GICD_ISENABLERN_5r; 1609 } 1610 1611 lkbde_irq_mask_set(sinfo->dev_no | LKBDE_ISR2_DEV | LKBDE_IPROC_REG, 1612 irq_mask_reg, mask, CMICX_TXRX_IRQ_MASK); 1613 } 1614 1615 static inline void 1616 xgsx_irq_mask_enable(bkn_switch_info_t *sinfo, int chan, int update_hw) 1617 { 1618 uint32_t mask; 1619 1620 if (CDMA_CH(sinfo, chan)) { 1621 mask = CMICX_DS_CMC_CTRLD_INT(chan); 1622 } else { 1623 if (chan == XGS_DMA_TX_CHAN) { 1624 mask = CMICX_DS_CMC_CHAIN_DONE(chan); 1625 } else { 1626 mask = CMICX_DS_CMC_DESC_DONE(chan) | 1627 CMICX_DS_CMC_CHAIN_DONE(chan); 1628 } 1629 } 1630 1631 sinfo->irq_mask |= mask; 1632 1633 if (update_hw) { 1634 xgsx_irq_mask_set(sinfo, sinfo->irq_mask); 1635 } 1636 } 1637 1638 static inline void 1639 xgsx_irq_mask_disable(bkn_switch_info_t *sinfo, int chan, int update_hw) 1640 { 1641 uint32_t mask; 1642 1643 if (CDMA_CH(sinfo, chan)) { 1644 mask = CMICX_DS_CMC_CTRLD_INT(chan); 1645 } else { 1646 if (chan == XGS_DMA_TX_CHAN) { 1647 mask = CMICX_DS_CMC_CHAIN_DONE(chan); 1648 } else { 1649 mask = CMICX_DS_CMC_DESC_DONE(chan) | 1650 CMICX_DS_CMC_CHAIN_DONE(chan); 1651 } 1652 } 1653 1654 sinfo->irq_mask &= ~mask; 1655 1656 if (update_hw) { 1657 xgsx_irq_mask_set(sinfo, sinfo->irq_mask); 1658 } 1659 } 1660 1661 static inline void 1662 dev_dma_chain_clear(bkn_switch_info_t *sinfo, int chan) 1663 { 1664 if (DEV_IS_CMICX(sinfo)) { 1665 xgsx_dma_chain_clear(sinfo, chan); 1666 } else if (DEV_IS_CMICM(sinfo)) { 1667 xgsm_dma_chain_clear(sinfo, chan); 1668 } else { 1669 xgs_dma_chain_clear(sinfo, chan); 1670 } 1671 } 1672 1673 static inline void 1674 dev_dma_desc_clear(bkn_switch_info_t *sinfo, int chan) 1675 { 1676 if (DEV_IS_CMICX(sinfo)) { 1677 xgsx_dma_desc_clear(sinfo, chan); 1678 } else if (DEV_IS_CMICM(sinfo)) { 1679 xgsm_dma_desc_clear(sinfo, chan); 1680 } else { 1681 xgs_dma_desc_clear(sinfo, chan); 1682 } 1683 } 1684 1685 static int 1686 dev_dma_chan_clear(bkn_switch_info_t *sinfo, int chan) 1687 { 1688 if (DEV_IS_CMICX(sinfo)) { 1689 return xgsx_dma_chan_clear(sinfo, chan); 1690 } else if (DEV_IS_CMICM(sinfo)) { 1691 return xgsm_dma_chan_clear(sinfo, chan); 1692 } else { 1693 return xgs_dma_chan_clear(sinfo, chan); 1694 } 1695 } 1696 1697 static void 1698 dev_cdma_halt_set(bkn_switch_info_t *sinfo, int chan) 1699 { 1700 if (DEV_IS_CMICX(sinfo)) { 1701 xgsx_cdma_halt_set(sinfo, chan); 1702 } else if (DEV_IS_CMICM(sinfo)) { 1703 xgsm_cdma_halt_set(sinfo, chan); 1704 } 1705 } 1706 1707 static int 1708 dev_dma_chan_init(bkn_switch_info_t *sinfo, int chan, int dir) 1709 { 1710 if (DEV_IS_CMICX(sinfo)) { 1711 return xgsx_dma_chan_init(sinfo, chan, dir); 1712 } else if (DEV_IS_CMICM(sinfo)) { 1713 return xgsm_dma_chan_init(sinfo, chan, dir); 1714 } else { 1715 return xgs_dma_chan_init(sinfo, chan, dir); 1716 } 1717 } 1718 1719 static int 1720 dev_dma_chan_start(bkn_switch_info_t *sinfo, int chan, uint64_t dcb) 1721 { 1722 if (DEV_IS_CMICX(sinfo)) { 1723 return xgsx_dma_chan_start(sinfo, chan, dcb); 1724 } else if (DEV_IS_CMICM(sinfo)) { 1725 return xgsm_dma_chan_start(sinfo, chan, dcb); 1726 } else { 1727 return xgs_dma_chan_start(sinfo, chan, dcb); 1728 } 1729 } 1730 1731 static int 1732 dev_dma_chan_abort(bkn_switch_info_t *sinfo, int chan, int polls) 1733 { 1734 if (DEV_IS_CMICX(sinfo)) { 1735 return xgsx_dma_chan_abort(sinfo, chan, polls); 1736 } else if (DEV_IS_CMICM(sinfo)) { 1737 return xgsm_dma_chan_abort(sinfo, chan, polls); 1738 } else { 1739 return xgs_dma_chan_abort(sinfo, chan, polls); 1740 } 1741 } 1742 1743 static inline void 1744 dev_irq_mask_set(bkn_switch_info_t *sinfo, uint32_t mask) 1745 { 1746 if (DEV_IS_CMICX(sinfo)) { 1747 xgsx_irq_mask_set(sinfo, mask); 1748 } else if (DEV_IS_CMICM(sinfo)) { 1749 xgsm_irq_mask_set(sinfo, mask); 1750 } else { 1751 xgs_irq_mask_set(sinfo, mask); 1752 } 1753 } 1754 1755 static inline void 1756 dev_irq_mask_enable(bkn_switch_info_t *sinfo, int chan, int update_hw) 1757 { 1758 if (DEV_IS_CMICX(sinfo)) { 1759 xgsx_irq_mask_enable(sinfo, chan, update_hw); 1760 } else if (DEV_IS_CMICM(sinfo)) { 1761 xgsm_irq_mask_enable(sinfo, chan, update_hw); 1762 } else { 1763 xgs_irq_mask_enable(sinfo, chan, update_hw); 1764 } 1765 } 1766 1767 static void 1768 dev_irq_mask_disable(bkn_switch_info_t *sinfo, int chan, int update_hw) 1769 { 1770 if (DEV_IS_CMICX(sinfo)) { 1771 xgsx_irq_mask_disable(sinfo, chan, update_hw); 1772 } else if (DEV_IS_CMICM(sinfo)) { 1773 xgsm_irq_mask_disable(sinfo, chan, update_hw); 1774 } else { 1775 xgs_irq_mask_disable(sinfo, chan, update_hw); 1776 } 1777 } 1778 1779 static int 1780 bkn_alloc_dcbs(bkn_switch_info_t *sinfo) 1781 { 1782 int dcb_size; 1783 int tx_ring_size, rx_ring_size; 1784 dma_addr_t dcb_dma = 0; 1785 1786 dcb_size = sinfo->dcb_wsize * sizeof(uint32_t); 1787 tx_ring_size = dcb_size * (MAX_TX_DCBS + 1); 1788 rx_ring_size = dcb_size * (MAX_RX_DCBS + 1); 1789 sinfo->dcb_mem_size = tx_ring_size + rx_ring_size * sinfo->rx_chans; 1790 1791 sinfo->dcb_mem = BKN_DMA_ALLOC_COHERENT(sinfo->dma_dev, 1792 sinfo->dcb_mem_size, 1793 &dcb_dma); 1794 if (sinfo->dcb_mem == NULL) { 1795 gprintk("DCB memory allocation (%d bytes) failed.\n", 1796 sinfo->dcb_mem_size); 1797 return -ENOMEM; 1798 } 1799 sinfo->dcb_dma = (uint64_t)dcb_dma; 1800 1801 return 0; 1802 } 1803 1804 static void 1805 bkn_free_dcbs(bkn_switch_info_t *sinfo) 1806 { 1807 if (sinfo->dcb_mem != NULL) { 1808 BKN_DMA_FREE_COHERENT(sinfo->dma_dev, sinfo->dcb_mem_size, 1809 sinfo->dcb_mem, (dma_addr_t)sinfo->dcb_dma); 1810 sinfo->dcb_mem = NULL; 1811 } 1812 } 1813 1814 static void 1815 bkn_clean_tx_dcbs(bkn_switch_info_t *sinfo) 1816 { 1817 bkn_desc_info_t *desc; 1818 1819 DBG_DCB_TX(("Cleaning Tx DCBs (%d %d).\n", 1820 sinfo->tx.cur, sinfo->tx.dirty)); 1821 while (sinfo->tx.free < MAX_TX_DCBS) { 1822 desc = &sinfo->tx.desc[sinfo->tx.dirty]; 1823 if (desc->skb != NULL) { 1824 DBG_SKB(("Cleaning Tx SKB from DCB %d.\n", 1825 sinfo->tx.dirty)); 1826 BKN_DMA_UNMAP_SINGLE(sinfo->dma_dev, 1827 desc->skb_dma, desc->dma_size, 1828 BKN_DMA_TODEV); 1829 desc->skb_dma = 0; 1830 dev_kfree_skb_any(desc->skb); 1831 desc->skb = NULL; 1832 } 1833 if (++sinfo->tx.dirty >= MAX_TX_DCBS) { 1834 sinfo->tx.dirty = 0; 1835 } 1836 sinfo->tx.free++; 1837 } 1838 sinfo->tx.api_active = 0; 1839 DBG_DCB_TX(("Cleaned Tx DCBs (%d %d).\n", 1840 sinfo->tx.cur, sinfo->tx.dirty)); 1841 } 1842 1843 static void 1844 bkn_clean_rx_dcbs(bkn_switch_info_t *sinfo, int chan) 1845 { 1846 bkn_desc_info_t *desc; 1847 1848 DBG_DCB_RX(("Cleaning Rx%d DCBs (%d %d).\n", 1849 chan, sinfo->rx[chan].cur, sinfo->rx[chan].dirty)); 1850 while (sinfo->rx[chan].free) { 1851 desc = &sinfo->rx[chan].desc[sinfo->rx[chan].dirty]; 1852 if (desc->skb != NULL) { 1853 DBG_SKB(("Cleaning Rx%d SKB from DCB %d.\n", 1854 chan, sinfo->rx[chan].dirty)); 1855 BKN_DMA_UNMAP_SINGLE(sinfo->dma_dev, 1856 desc->skb_dma, desc->dma_size, 1857 BKN_DMA_FROMDEV); 1858 desc->skb_dma = 0; 1859 dev_kfree_skb_any(desc->skb); 1860 desc->skb = NULL; 1861 } 1862 if (++sinfo->rx[chan].dirty >= MAX_RX_DCBS) { 1863 sinfo->rx[chan].dirty = 0; 1864 } 1865 sinfo->rx[chan].free--; 1866 } 1867 sinfo->rx[chan].running = 0; 1868 sinfo->rx[chan].api_active = 0; 1869 DBG_DCB_RX(("Cleaned Rx%d DCBs (%d %d).\n", 1870 chan, sinfo->rx[chan].cur, sinfo->rx[chan].dirty)); 1871 } 1872 1873 static void 1874 bkn_clean_dcbs(bkn_switch_info_t *sinfo) 1875 { 1876 int chan; 1877 1878 bkn_clean_tx_dcbs(sinfo); 1879 1880 for (chan = 0; chan < sinfo->rx_chans; chan++) { 1881 bkn_clean_rx_dcbs(sinfo, chan); 1882 } 1883 } 1884 1885 static void 1886 bkn_init_dcbs(bkn_switch_info_t *sinfo) 1887 { 1888 int dcb_size; 1889 uint32_t *dcb_mem; 1890 uint64_t dcb_dma; 1891 bkn_desc_info_t *desc; 1892 int idx; 1893 int chan; 1894 1895 memset(sinfo->dcb_mem, 0, sinfo->dcb_mem_size); 1896 1897 dcb_size = sinfo->dcb_wsize * sizeof(uint32_t); 1898 dcb_mem = sinfo->dcb_mem; 1899 dcb_dma = sinfo->dcb_dma; 1900 1901 for (idx = 0; idx < (MAX_TX_DCBS + 1); idx++) { 1902 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN)) { 1903 if (sinfo->cmic_type == 'x') { 1904 dcb_mem[2] |= 1 << 24 | 1 << 16; 1905 } else { 1906 dcb_mem[1] |= 1 << 24 | 1 << 16; 1907 } 1908 if (idx == MAX_TX_DCBS) { 1909 if (sinfo->cmic_type == 'x') { 1910 dcb_mem[0] = sinfo->tx.desc[0].dcb_dma; 1911 dcb_mem[1] = DMA_TO_BUS_HI(sinfo->tx.desc[0].dcb_dma >> 32); 1912 dcb_mem[2] |= 1 << 18; 1913 } else { 1914 dcb_mem[0] = sinfo->tx.desc[0].dcb_dma; 1915 dcb_mem[1] |= 1 << 18; 1916 } 1917 } 1918 } 1919 desc = &sinfo->tx.desc[idx]; 1920 desc->dcb_mem = dcb_mem; 1921 desc->dcb_dma = dcb_dma; 1922 dcb_mem += sinfo->dcb_wsize; 1923 dcb_dma += dcb_size; 1924 } 1925 sinfo->halt_addr[XGS_DMA_TX_CHAN] = sinfo->tx.desc[0].dcb_dma; 1926 sinfo->tx.free = MAX_TX_DCBS; 1927 sinfo->tx.cur = 0; 1928 sinfo->tx.dirty = 0; 1929 1930 DBG_DCB_TX(("Tx DCBs @ 0x%08x.\n", 1931 (uint32_t)sinfo->tx.desc[0].dcb_dma)); 1932 1933 for (chan = 0; chan < sinfo->rx_chans; chan++) { 1934 for (idx = 0; idx < (MAX_RX_DCBS + 1); idx++) { 1935 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 1936 if (sinfo->cmic_type == 'x') { 1937 dcb_mem[2] |= 1 << 24 | 1 << 16; 1938 } else { 1939 dcb_mem[1] |= 1 << 24 | 1 << 16; 1940 } 1941 if (idx == MAX_RX_DCBS) { 1942 if (sinfo->cmic_type == 'x') { 1943 dcb_mem[0] = sinfo->rx[chan].desc[0].dcb_dma; 1944 dcb_mem[1] = DMA_TO_BUS_HI(sinfo->rx[chan].desc[0].dcb_dma >> 32); 1945 dcb_mem[2] |= 1 << 18; 1946 } else { 1947 dcb_mem[0] = sinfo->rx[chan].desc[0].dcb_dma; 1948 dcb_mem[1] |= 1 << 18; 1949 } 1950 } 1951 } 1952 desc = &sinfo->rx[chan].desc[idx]; 1953 desc->dcb_mem = dcb_mem; 1954 desc->dcb_dma = dcb_dma; 1955 dcb_mem += sinfo->dcb_wsize; 1956 dcb_dma += dcb_size; 1957 } 1958 sinfo->halt_addr[XGS_DMA_RX_CHAN + chan] = sinfo->rx[chan].desc[MAX_RX_DCBS].dcb_dma; 1959 sinfo->rx[chan].free = 0; 1960 sinfo->rx[chan].cur = 0; 1961 sinfo->rx[chan].dirty = 0; 1962 1963 DBG_DCB_RX(("Rx%d DCBs @ 0x%08x.\n", 1964 chan, (uint32_t)sinfo->rx[chan].desc[0].dcb_dma)); 1965 } 1966 } 1967 1968 static void 1969 bkn_dump_dcb(char *prefix, uint32_t *dcb, int wsize, int txrx) 1970 { 1971 if (XGS_DMA_TX_CHAN == txrx) { 1972 if (wsize > 4) { 1973 DBG_DCB_TX(("%s: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x ... 0x%08x\n", 1974 prefix, dcb[0], dcb[1], dcb[2], dcb[3], dcb[4], dcb[5], 1975 dcb[wsize - 1])); 1976 } else { 1977 DBG_DCB_TX(("%s: 0x%08x 0x%08x 0x%08x 0x%08x\n", 1978 prefix, dcb[0], dcb[1], dcb[2], dcb[3])); 1979 } 1980 } else { 1981 if (wsize > 4) { 1982 DBG_DCB_RX(("%s: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x ... 0x%08x\n", 1983 prefix, dcb[0], dcb[1], dcb[2], dcb[3], dcb[4], dcb[5], 1984 dcb[wsize - 1])); 1985 } else { 1986 DBG_DCB_RX(("%s: 0x%08x 0x%08x 0x%08x 0x%08x\n", 1987 prefix, dcb[0], dcb[1], dcb[2], dcb[3])); 1988 } 1989 } 1990 } 1991 1992 static void 1993 bkn_dump_pkt(uint8_t *data, int size, int txrx) 1994 { 1995 int idx; 1996 char str[128]; 1997 1998 if (!(debug & DBG_LVL_PDMP) && 1999 !(txrx == XGS_DMA_TX_CHAN && debug & DBG_LVL_PDMP_TX) && 2000 !(txrx == XGS_DMA_RX_CHAN && debug & DBG_LVL_PDMP_RX)) { 2001 return; 2002 } 2003 2004 for (idx = 0; idx < size; idx++) { 2005 if ((idx & 0xf) == 0) { 2006 sprintf(str, "%04x: ", idx); 2007 } 2008 if ((idx & 0xf) == 8) { 2009 sprintf(&str[strlen(str)], "- "); 2010 } 2011 sprintf(&str[strlen(str)], "%02x ", data[idx]); 2012 if ((idx & 0xf) == 0xf) { 2013 sprintf(&str[strlen(str)], "\n"); 2014 gprintk(str); 2015 } 2016 } 2017 if ((idx & 0xf) != 0) { 2018 sprintf(&str[strlen(str)], "\n"); 2019 gprintk(str); 2020 } 2021 } 2022 2023 static bkn_switch_info_t * 2024 bkn_sinfo_from_unit(int unit) 2025 { 2026 struct list_head *list; 2027 bkn_switch_info_t *sinfo; 2028 2029 list_for_each(list, &_sinfo_list) { 2030 sinfo = (bkn_switch_info_t *)list; 2031 if (sinfo->dev_no == unit || unit == -1) { 2032 return sinfo; 2033 } 2034 } 2035 return NULL; 2036 } 2037 2038 static void 2039 bkn_cdma_goto(bkn_switch_info_t *sinfo, int chan, uint64_t dcb) 2040 { 2041 if (sinfo->basedev_suspended) { 2042 return; 2043 } 2044 2045 /* Set the new halt location */ 2046 sinfo->halt_addr[chan] = dcb; 2047 dev_cdma_halt_set(sinfo, chan); 2048 } 2049 2050 static void 2051 bkn_api_rx_restart(bkn_switch_info_t *sinfo, int chan) 2052 { 2053 bkn_dcb_chain_t *dcb_chain; 2054 int start_dma; 2055 2056 if (sinfo->basedev_suspended) { 2057 return; 2058 } 2059 2060 /* If resume from basedev suspended, there could be a suspended chain */ 2061 if (sinfo->rx[chan].api_dcb_chain) { 2062 return; 2063 } 2064 2065 /* Assume that driver lock is held */ 2066 if (!list_empty(&sinfo->rx[chan].api_dcb_list)) { 2067 dcb_chain = list_entry(sinfo->rx[chan].api_dcb_list.next, 2068 bkn_dcb_chain_t, list); 2069 start_dma = 0; 2070 if (sinfo->rx[chan].use_rx_skb == 0) { 2071 sinfo->rx[chan].chain_complete = 0; 2072 start_dma = 1; 2073 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan) && 2074 sinfo->rx[chan].api_active) { 2075 /* HW is running already, so we just move to the next chain */ 2076 start_dma = 0; 2077 } 2078 } 2079 sinfo->rx[chan].api_dcb_chain = dcb_chain; 2080 sinfo->rx[chan].api_active = 1; 2081 if (start_dma) { 2082 DBG_DCB_RX(("Start API Rx DMA, first DCB @ 0x%08x (%d DCBs).\n", 2083 (uint32_t)dcb_chain->dcb_dma, dcb_chain->dcb_cnt)); 2084 dev_dma_chan_clear(sinfo, XGS_DMA_RX_CHAN + chan); 2085 dev_irq_mask_enable(sinfo, XGS_DMA_RX_CHAN + chan, 1); 2086 dev_dma_chan_start(sinfo, XGS_DMA_RX_CHAN + chan, dcb_chain->dcb_dma); 2087 } 2088 2089 list_del(&dcb_chain->list); 2090 } 2091 } 2092 2093 static void 2094 bkn_api_rx_chain_done(bkn_switch_info_t *sinfo, int chan) 2095 { 2096 DBG_DCB_RX(("API Rx DMA chain done\n")); 2097 2098 if (!CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 2099 sinfo->rx[chan].api_active = 0; 2100 } 2101 if (sinfo->rx[chan].api_dcb_chain) { 2102 kfree(sinfo->rx[chan].api_dcb_chain); 2103 sinfo->rx[chan].api_dcb_chain = NULL; 2104 } 2105 bkn_api_rx_restart(sinfo, chan); 2106 } 2107 2108 static int 2109 bkn_api_rx_copy_from_skb(bkn_switch_info_t *sinfo, 2110 int chan, bkn_desc_info_t *desc) 2111 { 2112 bkn_dcb_chain_t *dcb_chain; 2113 uint32_t *dcb; 2114 uint32_t dcb_stat; 2115 uint8_t *pkt; 2116 uint64_t pkt_dma; 2117 int pktlen; 2118 int i; 2119 bkn_evt_resource_t *evt; 2120 2121 dcb_stat = desc->dcb_mem[sinfo->dcb_wsize-1]; 2122 pktlen = dcb_stat & SOC_DCB_KNET_COUNT_MASK; 2123 2124 dcb_chain = sinfo->rx[chan].api_dcb_chain; 2125 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan) && dcb_chain == NULL) { 2126 /* Last chain done, try to get a new one */ 2127 bkn_api_rx_chain_done(sinfo, chan); 2128 dcb_chain = sinfo->rx[chan].api_dcb_chain; 2129 } 2130 if (dcb_chain == NULL) { 2131 DBG_WARN(("No Rx API buffers\n")); 2132 sinfo->rx[chan].pkts_d_no_api_buf++; 2133 return -1; 2134 } 2135 dcb = &dcb_chain->dcb_mem[dcb_chain->dcb_cur * sinfo->dcb_wsize]; 2136 if ((sinfo->cmic_type == 'x' && (dcb[2] & 0xffff) < pktlen) || 2137 (sinfo->cmic_type != 'x' && (dcb[1] & 0xffff) < pktlen)) { 2138 DBG_WARN(("Rx API buffer too small\n")); 2139 return -1; 2140 } 2141 if (sinfo->cmic_type == 'x') { 2142 pkt_dma = BUS_TO_DMA_HI(dcb[1]); 2143 pkt_dma = pkt_dma << 32 | dcb[0]; 2144 } else { 2145 pkt_dma = dcb[0]; 2146 } 2147 pkt = (uint8_t *)kernel_bde->p2l(sinfo->dev_no, (sal_paddr_t)pkt_dma); 2148 if (pkt == NULL) { 2149 DBG_WARN(("Invalid Rx API buffer\n")); 2150 return -1; 2151 } 2152 2153 /* Copy packet data */ 2154 memcpy(pkt, desc->skb->data, pktlen); 2155 2156 /* Copy packet metadata and mark as done */ 2157 if (sinfo->cmic_type != 'x') { 2158 for (i = SOC_DCB_META_OFFSET; i < sinfo->dcb_wsize; i++) { 2159 dcb[i] = desc->dcb_mem[i]; 2160 } 2161 } 2162 dcb[sinfo->dcb_wsize-1] = dcb_stat | SOC_DCB_KNET_DONE; 2163 MEMORY_BARRIER; 2164 2165 dcb_chain->dcb_cur++; 2166 2167 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 2168 dcb = &dcb_chain->dcb_mem[dcb_chain->dcb_cur * sinfo->dcb_wsize]; 2169 if ((sinfo->cmic_type == 'x' && dcb[2] & (1 << 18)) || 2170 (sinfo->cmic_type != 'x' && dcb[1] & (1 << 18))) { 2171 /* Get the next chain if reload done */ 2172 dcb[sinfo->dcb_wsize-1] |= 1 << 31 | SOC_DCB_KNET_DONE; 2173 MEMORY_BARRIER; 2174 bkn_api_rx_chain_done(sinfo, chan); 2175 dcb_chain = sinfo->rx[chan].api_dcb_chain; 2176 if (dcb_chain == NULL) { 2177 sinfo->rx[chan].api_dcb_chain_end = NULL; 2178 } 2179 } 2180 } else { 2181 if ((sinfo->cmic_type == 'x' && (dcb[2] & (1 << 16)) == 0) || 2182 (sinfo->cmic_type != 'x' && (dcb[1] & (1 << 16)) == 0)) { 2183 bkn_api_rx_chain_done(sinfo, chan); 2184 } 2185 } 2186 2187 sinfo->dma_events |= KCOM_DMA_INFO_F_RX_DONE; 2188 2189 evt = &_bkn_evt[sinfo->evt_idx]; 2190 evt->evt_wq_put++; 2191 wake_up_interruptible(&evt->evt_wq); 2192 2193 return 0; 2194 } 2195 2196 static void 2197 bkn_rx_refill(bkn_switch_info_t *sinfo, int chan) 2198 { 2199 struct sk_buff *skb; 2200 bkn_desc_info_t *desc; 2201 uint32_t *dcb; 2202 uint32_t resv_size = sinfo->cmic_type == 'x' ? RCPU_HDR_SIZE : RCPU_RX_ENCAP_SIZE; 2203 uint32_t meta_size = sinfo->cmic_type == 'x' ? RCPU_RX_META_SIZE : 0; 2204 int prev; 2205 2206 if (sinfo->rx[chan].use_rx_skb == 0) { 2207 /* Rx buffers are provided by BCM Rx API */ 2208 return; 2209 } 2210 2211 if (!CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan) && 2212 sinfo->rx[chan].tokens < MAX_RX_DCBS) { 2213 /* Pause DMA for now */ 2214 return; 2215 } 2216 2217 while (sinfo->rx[chan].free < MAX_RX_DCBS) { 2218 desc = &sinfo->rx[chan].desc[sinfo->rx[chan].cur]; 2219 if (desc->skb == NULL) { 2220 skb = dev_alloc_skb(rx_buffer_size + RCPU_RX_ENCAP_SIZE); 2221 if (skb == NULL) { 2222 break; 2223 } 2224 skb_reserve(skb, resv_size); 2225 desc->skb = skb; 2226 } else { 2227 DBG_DCB_RX(("Refill Rx%d SKB in DCB %d recycled.\n", 2228 chan, sinfo->rx[chan].cur)); 2229 } 2230 skb = desc->skb; 2231 desc->dma_size = rx_buffer_size + meta_size; 2232 #ifdef KNET_NO_AXI_DMA_INVAL 2233 /* 2234 * FIXME: Need to retain this code until iProc customers have been 2235 * migrated to updated u-boot. Old u-boot versions are unable to load 2236 * the kernel into non-ACP memory. 2237 */ 2238 /* 2239 * Cache invalidate may corrupt DMA memory on some iProc-based devices 2240 * if the kernel is mapped to ACP memory. 2241 */ 2242 if (sinfo->pdev == NULL) { 2243 desc->dma_size = 0; 2244 } 2245 #endif 2246 desc->skb_dma = BKN_DMA_MAP_SINGLE(sinfo->dma_dev, 2247 skb->data, desc->dma_size, 2248 BKN_DMA_FROMDEV); 2249 if (BKN_DMA_MAPPING_ERROR(sinfo->dma_dev, desc->skb_dma)) { 2250 dev_kfree_skb_any(skb); 2251 desc->skb = NULL; 2252 break; 2253 } 2254 DBG_DCB_RX(("Refill Rx%d DCB %d (0x%08x).\n", 2255 chan, sinfo->rx[chan].cur, (uint32_t)desc->skb_dma)); 2256 dcb = desc->dcb_mem; 2257 dcb[0] = desc->skb_dma; 2258 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 2259 if (sinfo->cmic_type == 'x') { 2260 dcb[2] |= 1 << 24 | 1 << 16; 2261 } else { 2262 dcb[1] |= 1 << 24 | 1 << 16; 2263 } 2264 } else { 2265 prev = PREV_IDX(sinfo->rx[chan].cur, MAX_RX_DCBS); 2266 if (prev < (MAX_RX_DCBS - 1)) { 2267 if (sinfo->cmic_type == 'x') { 2268 sinfo->rx[chan].desc[prev].dcb_mem[2] |= 1 << 16; 2269 } else { 2270 sinfo->rx[chan].desc[prev].dcb_mem[1] |= 1 << 16; 2271 } 2272 } 2273 } 2274 if (sinfo->cmic_type == 'x') { 2275 dcb[1] = DMA_TO_BUS_HI(desc->skb_dma >> 32); 2276 dcb[2] |= rx_buffer_size + meta_size; 2277 } else { 2278 dcb[1] |= rx_buffer_size; 2279 } 2280 2281 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan) && 2282 sinfo->rx[chan].tokens > MAX_RX_DCBS) { 2283 /* DMA run to the new halt location */ 2284 bkn_cdma_goto(sinfo, XGS_DMA_RX_CHAN + chan, desc->dcb_dma); 2285 } 2286 2287 if (++sinfo->rx[chan].cur >= MAX_RX_DCBS) { 2288 sinfo->rx[chan].cur = 0; 2289 } 2290 sinfo->rx[chan].free++; 2291 sinfo->rx[chan].tokens--; 2292 } 2293 } 2294 2295 static int 2296 bkn_rx_restart(bkn_switch_info_t *sinfo, int chan) 2297 { 2298 bkn_desc_info_t *desc; 2299 2300 if (sinfo->basedev_suspended) { 2301 return 0; 2302 } 2303 2304 if (sinfo->rx[chan].running) { 2305 return 0; 2306 } 2307 2308 if (sinfo->rx[chan].free < MAX_RX_DCBS) { 2309 return 1; 2310 } 2311 2312 desc = &sinfo->rx[chan].desc[sinfo->rx[chan].dirty]; 2313 sinfo->rx[chan].chain_complete = 0; 2314 DBG_DCB_RX(("Restart Rx%d DMA, DCB @ 0x%08x (%d).\n", 2315 chan, (uint32_t)desc->dcb_dma, sinfo->rx[chan].dirty)); 2316 dev_dma_chan_clear(sinfo, XGS_DMA_RX_CHAN + chan); 2317 dev_irq_mask_enable(sinfo, XGS_DMA_RX_CHAN + chan, 1); 2318 dev_dma_chan_start(sinfo, XGS_DMA_RX_CHAN + chan, desc->dcb_dma); 2319 sinfo->rx[chan].running = 1; 2320 2321 /* Request one extra poll if chain was restarted during poll */ 2322 if (sinfo->napi_poll_mode) { 2323 sinfo->napi_poll_again = 1; 2324 } 2325 2326 return 0; 2327 } 2328 2329 static int 2330 bkn_tx_dma_start(bkn_switch_info_t *sinfo) 2331 { 2332 bkn_desc_info_t *desc; 2333 2334 desc = &sinfo->tx.desc[sinfo->tx.cur]; 2335 if (sinfo->tx.free == MAX_TX_DCBS) { 2336 if (!sinfo->tx.api_active) { 2337 DBG_DCB_TX(("Start Tx DMA, DCB @ 0x%08x (%d).\n", 2338 (uint32_t)desc->dcb_dma, sinfo->tx.cur)); 2339 dev_dma_chan_clear(sinfo, XGS_DMA_TX_CHAN); 2340 dev_irq_mask_enable(sinfo, XGS_DMA_TX_CHAN, 1); 2341 dev_dma_chan_start(sinfo, XGS_DMA_TX_CHAN, desc->dcb_dma); 2342 } 2343 } 2344 2345 return 0; 2346 } 2347 2348 static int 2349 bkn_dma_init(bkn_switch_info_t *sinfo) 2350 { 2351 int chan; 2352 2353 dev_dma_chan_init(sinfo, XGS_DMA_TX_CHAN, 1); 2354 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN)) { 2355 bkn_tx_dma_start(sinfo); 2356 } 2357 2358 for (chan = 0; chan < sinfo->rx_chans; chan++) { 2359 dev_dma_chan_init(sinfo, XGS_DMA_RX_CHAN + chan, 0); 2360 bkn_rx_refill(sinfo, chan); 2361 bkn_rx_restart(sinfo, chan); 2362 } 2363 2364 return 0; 2365 } 2366 2367 static int 2368 bkn_dma_abort_tx(bkn_switch_info_t *sinfo) 2369 { 2370 bkn_dcb_chain_t *dcb_chain; 2371 2372 DBG_VERB(("Aborting Tx DMA.\n")); 2373 2374 dev_irq_mask_disable(sinfo, XGS_DMA_TX_CHAN, 1); 2375 2376 dev_dma_chan_abort(sinfo, XGS_DMA_TX_CHAN, 10000); 2377 2378 if (sinfo->tx.api_dcb_chain) { 2379 DBG_DCB_TX(("Freeing active Tx DCB chain.\n")); 2380 kfree(sinfo->tx.api_dcb_chain); 2381 sinfo->tx.api_dcb_chain = NULL; 2382 } 2383 while (!list_empty(&sinfo->tx.api_dcb_list)) { 2384 dcb_chain = list_entry(sinfo->tx.api_dcb_list.next, 2385 bkn_dcb_chain_t, list); 2386 list_del(&dcb_chain->list); 2387 DBG_DCB_TX(("Freeing Tx DCB chain.\n")); 2388 kfree(dcb_chain); 2389 } 2390 sinfo->tx.api_dcb_chain_end = NULL; 2391 2392 return 0; 2393 } 2394 2395 static int 2396 bkn_dma_abort_rx(bkn_switch_info_t *sinfo, int chan) 2397 { 2398 bkn_dcb_chain_t *dcb_chain; 2399 2400 DBG_VERB(("Aborting Rx%d DMA.\n", chan)); 2401 2402 dev_irq_mask_disable(sinfo, XGS_DMA_RX_CHAN + chan, 1); 2403 2404 dev_dma_chan_abort(sinfo, XGS_DMA_RX_CHAN + chan, 10000); 2405 2406 if (sinfo->rx[chan].api_dcb_chain) { 2407 DBG_DCB_RX(("Freeing active Rx%d DCB chain.\n", chan)); 2408 kfree(sinfo->rx[chan].api_dcb_chain); 2409 sinfo->rx[chan].api_dcb_chain = NULL; 2410 } 2411 while (!list_empty(&sinfo->rx[chan].api_dcb_list)) { 2412 dcb_chain = list_entry(sinfo->rx[chan].api_dcb_list.next, 2413 bkn_dcb_chain_t, list); 2414 list_del(&dcb_chain->list); 2415 DBG_DCB_RX(("Freeing Rx%d DCB chain.\n", chan)); 2416 kfree(dcb_chain); 2417 } 2418 sinfo->rx[chan].api_dcb_chain_end = NULL; 2419 2420 return 0; 2421 } 2422 2423 static int 2424 bkn_dma_abort(bkn_switch_info_t *sinfo) 2425 { 2426 int chan; 2427 2428 bkn_dma_abort_tx(sinfo); 2429 2430 for (chan = 0; chan < sinfo->rx_chans; chan++) { 2431 bkn_dma_abort_rx(sinfo, chan); 2432 } 2433 2434 return 0; 2435 } 2436 2437 static int 2438 device_is_dpp(bkn_switch_info_t *sinfo) 2439 { 2440 int is_dpp = 0; 2441 2442 is_dpp = (sinfo->dcb_type == 28) ? 1 : 0; 2443 return is_dpp; 2444 } 2445 2446 static int 2447 device_is_dnx(bkn_switch_info_t *sinfo) 2448 { 2449 int is_dnx = 0; 2450 2451 is_dnx = (sinfo->dcb_type == 39) ? 1 : 0; 2452 return is_dnx; 2453 } 2454 2455 /* is DPP or is DNX*/ 2456 static int 2457 device_is_sand(bkn_switch_info_t *sinfo) 2458 { 2459 int is_sand = 0; 2460 2461 if (device_is_dpp(sinfo) || device_is_dnx(sinfo)) { 2462 is_sand = 1; 2463 } 2464 return is_sand; 2465 } 2466 2467 static bkn_filter_t * 2468 bkn_match_rx_pkt(bkn_switch_info_t *sinfo, uint8_t *pkt, int pktlen, 2469 void *meta, int chan, bkn_filter_t *cbf) 2470 { 2471 struct list_head *list; 2472 bkn_filter_t *filter; 2473 kcom_filter_t scratch, *kf; 2474 uint8_t *oob = (uint8_t *)meta; 2475 int size, wsize; 2476 int idx, match; 2477 2478 list_for_each(list, &sinfo->rxpf_list) { 2479 filter = (bkn_filter_t *)list; 2480 kf = &filter->kf; 2481 memcpy(&scratch.data.b[0], 2482 &oob[kf->oob_data_offset], kf->oob_data_size); 2483 memcpy(&scratch.data.b[kf->oob_data_size], 2484 &pkt[kf->pkt_data_offset], kf->pkt_data_size); 2485 size = kf->oob_data_size + kf->pkt_data_size; 2486 wsize = BYTES2WORDS(size); 2487 DBG_VERB(("Filter: size = %d (%d), data = 0x%08x, mask = 0x%08x\n", 2488 size, wsize, kf->data.w[0], kf->mask.w[0])); 2489 2490 if (device_is_sand(sinfo)) { 2491 DBG_DUNE(("Filter: size = %d (wsize %d)\n", size, wsize)); 2492 for (idx = 0; idx < wsize; idx++) 2493 { 2494 DBG_DUNE(("OOB[%d]: 0x%08x [0x%08x]\n", idx, kf->data.w[idx], kf->mask.w[idx])); 2495 } 2496 DBG_DUNE(("Meta Data [+ Selected Raw packet data]\n")); 2497 for (idx = 0; idx < wsize; idx++) 2498 { 2499 DBG_DUNE(("Scratch[%d]: 0x%08x\n", idx, scratch.data.w[idx])); 2500 } 2501 } 2502 2503 match = 1; 2504 if (match) { 2505 if (device_is_dnx(sinfo)) 2506 { 2507 /* 2508 * Mutliple RX channels are enabled on JR2 and above devices 2509 * Bind between priority 0 and RX channel 0 is not checked, then all enabled RX channels can receive packets. 2510 */ 2511 if (kf->priority && (kf->priority < (num_rx_prio * sinfo->rx_chans))) { 2512 if (kf->priority < (num_rx_prio * chan) || 2513 kf->priority >= (num_rx_prio * (chan + 1))) { 2514 match = 0; 2515 } 2516 } 2517 } 2518 else { 2519 if (kf->priority < (num_rx_prio * sinfo->rx_chans)) { 2520 if (kf->priority < (num_rx_prio * chan) || 2521 kf->priority >= (num_rx_prio * (chan + 1))) { 2522 match = 0; 2523 } 2524 } 2525 } 2526 } 2527 if (match) { 2528 for (idx = 0; idx < wsize; idx++) { 2529 scratch.data.w[idx] &= kf->mask.w[idx]; 2530 if (scratch.data.w[idx] != kf->data.w[idx]) { 2531 match = 0; 2532 break; 2533 } 2534 } 2535 } 2536 if (match) { 2537 if (kf->dest_type == KCOM_DEST_T_CB) { 2538 /* Check for custom filters */ 2539 if (knet_filter_cb != NULL && cbf != NULL) { 2540 memset(cbf, 0, sizeof(*cbf)); 2541 memcpy(&cbf->kf, kf, sizeof(cbf->kf)); 2542 if (knet_filter_cb(pkt, pktlen, sinfo->dev_no, 2543 meta, chan, &cbf->kf)) { 2544 filter->hits++; 2545 return cbf; 2546 } 2547 } else { 2548 DBG_FLTR(("Match, but not filter callback\n")); 2549 } 2550 } else { 2551 filter->hits++; 2552 return filter; 2553 } 2554 } 2555 } 2556 2557 return NULL; 2558 } 2559 2560 static bkn_priv_t * 2561 bkn_netif_lookup(bkn_switch_info_t *sinfo, int id) 2562 { 2563 struct list_head *list; 2564 bkn_priv_t *priv; 2565 int found; 2566 2567 /* Fast path */ 2568 if (id < sinfo->ndev_max) { 2569 if (sinfo->ndevs[id] != NULL) { 2570 DBG_NDEV(("Look up netif ID %d successful\n", id)); 2571 return netdev_priv(sinfo->ndevs[id]); 2572 } 2573 } 2574 2575 /* Slow path - should normally not get here */ 2576 found = 0; 2577 priv = NULL; 2578 list_for_each(list, &sinfo->ndev_list) { 2579 priv = (bkn_priv_t *)list; 2580 if (priv->id == id) { 2581 found = 1; 2582 break; 2583 } 2584 } 2585 if (found && priv != NULL) { 2586 return priv; 2587 } 2588 return NULL; 2589 } 2590 2591 static int 2592 bkn_hw_tstamp_rx_set(bkn_switch_info_t *sinfo, int phys_port, struct sk_buff *skb, uint32 *meta) 2593 { 2594 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 2595 uint64_t ts = 0; 2596 2597 2598 if (knet_hw_tstamp_rx_time_upscale_cb) { 2599 if (knet_hw_tstamp_rx_time_upscale_cb(sinfo->dev_no, phys_port, skb, meta, &ts) < 0) { 2600 return -1; 2601 } 2602 } 2603 2604 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 2605 shhwtstamps->hwtstamp = ns_to_ktime(ts); 2606 2607 return 0; 2608 } 2609 2610 static int 2611 bkn_add_rcpu_encap(bkn_switch_info_t *sinfo, struct sk_buff *skb, void *meta, int len) 2612 { 2613 int pktlen = skb->len; 2614 uint32_t *dmeta, *smeta, wsize, psize; 2615 int idx; 2616 2617 /* Add and clear RCPU encapsulation */ 2618 if (device_is_sand(sinfo)) { 2619 psize = RCPU_RX_ENCAP_SIZE; 2620 skb_push(skb, psize); 2621 memset(skb->data, 0, RCPU_RX_ENCAP_SIZE); 2622 } else if (sinfo->cmic_type == 'x') { 2623 psize = RCPU_HDR_SIZE + sinfo->pkt_hdr_size; 2624 skb_push(skb, psize); 2625 memset(skb->data, 0, RCPU_HDR_SIZE); 2626 } else { 2627 psize = RCPU_RX_ENCAP_SIZE; 2628 skb_push(skb, psize); 2629 memset(skb->data, 0, RCPU_RX_ENCAP_SIZE); 2630 } 2631 2632 /* RCPU Header */ 2633 memcpy(skb->data, &skb->data[psize], 12); 2634 if (rcpu_dmac != NULL) { 2635 memcpy(skb->data, bkn_rcpu_dmac, 6); 2636 } 2637 if (rcpu_smac != NULL) { 2638 memcpy(&skb->data[6], bkn_rcpu_smac, 6); 2639 } 2640 skb->data[12] = 0x81; 2641 skb->data[14] = rcpu_vlan >> 8; 2642 skb->data[15] = rcpu_vlan & 0xff; 2643 skb->data[16] = rcpu_ethertype >> 8; 2644 skb->data[17] = rcpu_ethertype & 0xff; 2645 skb->data[18] = sinfo->rcpu_sig >> 8; 2646 skb->data[19] = sinfo->rcpu_sig & 0xff; 2647 skb->data[20] = RCPU_OPCODE_RX; 2648 skb->data[21] = RCPU_F_MODHDR; 2649 skb->data[24] = pktlen >> 8; 2650 skb->data[25] = pktlen & 0xff; 2651 2652 /* Meta data */ 2653 dmeta = (uint32_t *)&skb->data[RCPU_HDR_SIZE]; 2654 2655 if (device_is_sand(sinfo)) { 2656 /* Copy at most 64 bytes system headers */ 2657 len = len > RCPU_RX_META_SIZE ? RCPU_RX_META_SIZE : len; 2658 memcpy(&skb->data[RCPU_HDR_SIZE], (uint8_t *)meta, len); 2659 } else { 2660 smeta = sinfo->cmic_type == 'x' ? (uint32_t *)meta : (uint32_t *)meta + 2; 2661 wsize = sinfo->cmic_type == 'x' ? sinfo->pkt_hdr_size / 4 : sinfo->dcb_wsize - 3; 2662 for (idx = 0; idx < wsize; idx++) { 2663 dmeta[idx] = htonl(smeta[idx]); 2664 } 2665 } 2666 2667 2668 return 0; 2669 } 2670 2671 static void 2672 bkn_eth_type_update(struct sk_buff *skb, int ethertype) 2673 { 2674 #if defined(PM_ETH_TYPE) && defined(PM_FC_TYPE) 2675 /* Optionally override standard protocol */ 2676 skb->protocol = PM_ETH_TYPE; 2677 if (ethertype == ETH_P_FCOE) { 2678 skb->protocol = PM_FC_TYPE; 2679 } 2680 #endif 2681 } 2682 2683 #define BKN_DNX_BIT(x) (1<<(x)) 2684 #define BKN_DNX_RBIT(x) (~(1<<(x))) 2685 #ifdef __LITTLE_ENDIAN 2686 #define BKN_DNX_BYTE_SWAP(x) (x) 2687 #else 2688 #define BKN_DNX_BYTE_SWAP(x) ((((x) << 24)) | (((x) & 0xff00) << 8) | (((x) & 0xff0000) >> 8) | (((x) >> 24))) 2689 #endif 2690 2691 static int 2692 packet_is_untagged(uint16_t tpid) 2693 { 2694 int is_untagged = 0; 2695 2696 /* 0x8100 is used in 802.1Q */ 2697 /* 0x8848 is used in 802.11ad, the dtag tpid can be set to anything besides 0x8848, 0x9100 is a typical value, but couldn't cover all scenarios. */ 2698 is_untagged = ((tpid != 0x8100) && (tpid != 0x8848) && (tpid != 0x9100)); 2699 return is_untagged; 2700 } 2701 2702 static void 2703 bkn_bitstream_set_field(uint32_t *input_buffer, uint32_t start_bit, uint32_t nof_bits, uint32_t field) 2704 { 2705 uint32_t place; 2706 uint32_t field_bit_i; 2707 uint32_t bit_indicator; 2708 2709 if( nof_bits > 32) 2710 { 2711 return; 2712 } 2713 2714 for( place=start_bit, field_bit_i = 0; field_bit_i< nof_bits; ++place, ++field_bit_i) 2715 { 2716 bit_indicator = field & BKN_DNX_BIT(nof_bits-field_bit_i-1); 2717 if(bit_indicator) 2718 { 2719 input_buffer[place>>5] |= (0x80000000 >> (place & 0x0000001F)); 2720 } 2721 else 2722 { 2723 input_buffer[place>>5] &= ~(0x80000000 >> (place & 0x0000001F)); 2724 } 2725 } 2726 return; 2727 } 2728 2729 static void 2730 bkn_bitstream_get_field(uint8_t *input_buffer, uint32_t start_bit, uint32_t nof_bits, uint32_t *output_value) 2731 { 2732 uint32_t idx; 2733 uint32_t buf_sizes=0; 2734 uint32_t tmp_output_value[2]={0}; 2735 uint32_t first_byte_ndx; 2736 uint32_t last_byte_ndx; 2737 uint32_t place; 2738 uint32_t field_bit_i; 2739 uint8_t *tmp_output_value_u8_ptr = (uint8_t*)&tmp_output_value; 2740 uint32_t bit_indicator; 2741 2742 if (nof_bits > 32) 2743 { 2744 return; 2745 } 2746 2747 first_byte_ndx = start_bit / 8; 2748 last_byte_ndx = ((start_bit + nof_bits - 1) / 8); 2749 *output_value=0; 2750 2751 /* get 32 bit value, MSB */ 2752 for (idx = first_byte_ndx; idx <= last_byte_ndx; ++idx) 2753 { 2754 tmp_output_value_u8_ptr[last_byte_ndx - idx] = input_buffer[idx]; 2755 buf_sizes += 8; 2756 } 2757 tmp_output_value[0] = BKN_DNX_BYTE_SWAP(tmp_output_value[0]); 2758 if (last_byte_ndx > 4) 2759 { 2760 tmp_output_value[1] = BKN_DNX_BYTE_SWAP(tmp_output_value[1]); 2761 } 2762 2763 place = buf_sizes - (start_bit % 8 + nof_bits); 2764 for (field_bit_i = 0; field_bit_i< nof_bits; ++place, ++field_bit_i) 2765 { 2766 uint32_t result; 2767 result = tmp_output_value[place>>5] & BKN_DNX_BIT(place & 0x0000001F); 2768 if (result) 2769 { 2770 bit_indicator = 1; 2771 } else { 2772 bit_indicator = 0; 2773 } 2774 *output_value |= bit_indicator << field_bit_i; 2775 } 2776 return; 2777 } 2778 2779 static void 2780 bkn_dpp_packet_parse_ftmh( 2781 bkn_switch_info_t *sinfo, 2782 uint8_t *buf, 2783 uint32_t buf_len, 2784 bkn_dune_system_header_info_t *packet_info, 2785 uint8_t *is_tsh_en, 2786 uint8_t *is_inter_hdr_en) 2787 { 2788 uint32_t pkt_offset = packet_info->system_header_size; 2789 uint32_t dsp_ext_exist = 0; 2790 uint32_t fld_val; 2791 2792 /* FTMH: Source-system-port-aggregate */ 2793 bkn_bitstream_get_field( 2794 &buf[pkt_offset], 2795 BKN_DPP_FTMH_SRC_SYS_PORT_MSB, 2796 BKN_DPP_FTMH_SRC_SYS_PORT_NOF_BITS, 2797 &packet_info->ftmh.source_sys_port_aggregate); 2798 /* FTMH: TM-action-type */ 2799 bkn_bitstream_get_field( 2800 &buf[pkt_offset], 2801 BKN_DPP_FTMH_ACTION_TYPE_MSB, 2802 BKN_DPP_FTMH_ACTION_TYPE_NOF_BITS, 2803 &packet_info->ftmh.action_type); 2804 /* FTMH: Internal-type */ 2805 bkn_bitstream_get_field( 2806 &buf[pkt_offset], 2807 BKN_DPP_FTMH_PPH_TYPE_MSB, 2808 BKN_DPP_FTMH_PPH_TYPE_NOF_BITS, 2809 &fld_val); 2810 2811 /* FTMH: DSP Extension */ 2812 bkn_bitstream_get_field( 2813 &buf[pkt_offset], 2814 BKN_DPP_FTMH_EXT_DSP_EXIST_MSB, 2815 BKN_DPP_FTMH_EXT_DSP_EXIST_NOF_BITS, 2816 &dsp_ext_exist); 2817 2818 if (fld_val & 0x1) 2819 { 2820 *is_inter_hdr_en = TRUE; 2821 } 2822 if (fld_val & 0x2) 2823 { 2824 *is_tsh_en = TRUE; 2825 } 2826 2827 pkt_offset += BKN_DPP_FTMH_SIZE_BYTE; 2828 DBG_DUNE(("FTMH(9-%u): Action-type %d Source-system-port 0x%x\n", pkt_offset, 2829 packet_info->ftmh.action_type, 2830 packet_info->ftmh.source_sys_port_aggregate)); 2831 2832 /* FTMH LB-Key Extension */ 2833 if (sinfo->ftmh_lb_key_ext_size) 2834 { 2835 pkt_offset += sinfo->ftmh_lb_key_ext_size; 2836 DBG_DUNE(("FTMH LB-Key Extension(1-%u) is present\n", pkt_offset)); 2837 } 2838 2839 if (dsp_ext_exist) 2840 { 2841 pkt_offset += BKN_DPP_FTMH_DEST_EXT_SIZE_BYTE; 2842 DBG_DUNE(("FTMH DSP Extension(2-%u) is present\n", pkt_offset)); 2843 } 2844 2845 /* FTMH Stacking Extension */ 2846 if (sinfo->ftmh_stacking_ext_size) 2847 { 2848 pkt_offset += sinfo->ftmh_stacking_ext_size; 2849 DBG_DUNE(("FTMH Stacking Extension(2-%u) is present\n", pkt_offset)); 2850 } 2851 2852 packet_info->system_header_size = pkt_offset; 2853 return; 2854 } 2855 2856 static void 2857 bkn_dpp_packet_parse_otsh( 2858 bkn_switch_info_t *sinfo, 2859 uint8_t *buf, 2860 uint32_t buf_len, 2861 bkn_dune_system_header_info_t *packet_info, 2862 uint8_t *is_oam_dm_tod_en, 2863 uint8_t *is_skip_udh) 2864 { 2865 uint32_t pkt_offset = packet_info->system_header_size; 2866 uint32_t type = 0; 2867 uint32_t oam_sub_type = 0; 2868 2869 /* OTSH: TYPE */ 2870 bkn_bitstream_get_field( 2871 &buf[pkt_offset], 2872 BKN_DPP_OTSH_TYPE_MSB, 2873 BKN_DPP_OTSH_TYPE_NOF_BITS, 2874 &type); 2875 if (type == BKN_DPP_OTSH_TYPE_OAM) { 2876 /* OTSH: OAM_SUB_TYPE */ 2877 bkn_bitstream_get_field( 2878 &buf[pkt_offset], 2879 BKN_DPP_OTSH_OAM_SUB_TYPE_MSB, 2880 BKN_DPP_OTSH_OAM_SUB_TYPE_NOF_BITS, 2881 &oam_sub_type); 2882 if ((oam_sub_type == BKN_DPP_OTSH_OAM_SUB_TYPE_DM_1588) || (oam_sub_type == BKN_DPP_OTSH_OAM_SUB_TYPE_DM_NTP)) { 2883 *is_oam_dm_tod_en = TRUE; 2884 /* Down MEP DM trapped packets will not have UDH present (even if configured), except for QAX when custom_feature_oam_dm_tod_msb_add_enable=0 */ 2885 if (!sinfo->no_skip_udh_check) { 2886 *is_skip_udh = TRUE; 2887 } 2888 } 2889 } 2890 packet_info->system_header_size += BKN_DPP_OTSH_SIZE_BYTE; 2891 2892 DBG_DUNE(("OTSH(%d): Type 0x%x OAM-Sub-Type 0x%x\n", BKN_DPP_OTSH_SIZE_BYTE, type, oam_sub_type)); 2893 return; 2894 } 2895 2896 static void 2897 bkn_dpp_packet_parse_internal( 2898 bkn_switch_info_t *sinfo, 2899 uint8_t *buf, 2900 uint32_t buf_len, 2901 bkn_dune_system_header_info_t *packet_info, 2902 uint8_t is_oamp_punted, 2903 uint8_t *is_trapped) 2904 { 2905 uint32_t pkt_offset = packet_info->system_header_size; 2906 uint32_t fld_val = 0; 2907 uint32_t eei_extension_present = 0; 2908 uint32_t learn_extension_present = 0; 2909 uint32_t fhei_size = 0; 2910 2911 /* Internal: EEI EXT */ 2912 bkn_bitstream_get_field( 2913 &buf[pkt_offset], 2914 BKN_DPP_INTERNAL_EEI_EXTENSION_PRESENT_MSB, 2915 BKN_DPP_INTERNAL_EEI_EXTENSION_PRESENT_NOF_BITS, 2916 &eei_extension_present); 2917 /* Internal: LERAN EXT */ 2918 bkn_bitstream_get_field( 2919 &buf[pkt_offset], 2920 BKN_DPP_INTERNAL_LEARN_EXENSION_PRESENT_MSB, 2921 BKN_DPP_INTERNAL_LEARN_EXENSION_PRESENT_NOF_BITS, 2922 &learn_extension_present); 2923 /* Internal: FHEI EXT */ 2924 bkn_bitstream_get_field( 2925 &buf[pkt_offset], 2926 BKN_DPP_INTERNAL_FHEI_SIZE_MSB, 2927 BKN_DPP_INTERNAL_FHEI_SIZE_NOF_BITS, 2928 &fhei_size); 2929 /* Internal: FORWARD_CODE */ 2930 bkn_bitstream_get_field( 2931 &buf[pkt_offset], 2932 BKN_DPP_INTERNAL_FORWARD_CODE_MSB, 2933 BKN_DPP_INTERNAL_FORWARD_CODE_NOF_BITS, 2934 &fld_val); 2935 *is_trapped = (uint8_t)(fld_val == BKN_DPP_INTERNAL_FORWARD_CODE_CPU_TRAP); 2936 /* Internal: VSI */ 2937 bkn_bitstream_get_field( 2938 &buf[pkt_offset], 2939 BKN_DPP_INTERNAL_VSI_MSB, 2940 BKN_DPP_INTERNAL_VSI_NOF_BITS, 2941 &fld_val); 2942 packet_info->internal.forward_domain = fld_val; 2943 2944 if (is_oamp_punted && *is_trapped) { 2945 if(fhei_size == 3) { 2946 /* Force to FHEI size 1 when packets are punted by OAMP */ 2947 fhei_size = 1; 2948 } 2949 } 2950 2951 /* Move forward to end of Internal header */ 2952 pkt_offset += BKN_DPP_INTERNAL_SIZE_BYTE; 2953 2954 DBG_DUNE(("PPH(7-%u): EEI-Extension %d Learn-Extension %d VSI %d FHEI-size %d\n", 2955 pkt_offset, eei_extension_present,learn_extension_present, packet_info->internal.forward_domain, fhei_size)); 2956 2957 /* Advance header according to FHEI Trap extension */ 2958 switch(fhei_size) { 2959 case 1: 2960 /* 3B FHEI Extension: do nothing, header poniter is in the right location */ 2961 break; 2962 case 2: 2963 /* 5B FHEI Extension: adavance header pointer in 2B */ 2964 pkt_offset += 2; 2965 break; 2966 case 3: 2967 /* 8B FHEI Extension: adavance header pointer in 5B */ 2968 pkt_offset += 5; 2969 break; 2970 default: 2971 break; 2972 } 2973 2974 /* Internal extension */ 2975 if (*is_trapped && fhei_size) 2976 { 2977 /* CPU trap code qualifier */ 2978 bkn_bitstream_get_field( 2979 &buf[pkt_offset], 2980 BKN_DPP_INTERNAL_FHEI_TRAP_SNOOP_3B_CPU_TRAP_CODE_QUALIFIER_MSB, 2981 BKN_DPP_INTERNAL_FHEI_TRAP_SNOOP_3B_CPU_TRAP_CODE_QUALIFIER_NOF_BITS, 2982 &packet_info->internal.trap_qualifier); 2983 /* CPU trap code */ 2984 bkn_bitstream_get_field( 2985 &buf[pkt_offset], 2986 BKN_DPP_INTERNAL_FHEI_TRAP_SNOOP_3B_CPU_TRAP_CODE_MSB, 2987 BKN_DPP_INTERNAL_FHEI_TRAP_SNOOP_3B_CPU_TRAP_CODE_NOF_BITS, 2988 &packet_info->internal.trap_id); 2989 2990 DBG_DUNE(("FHEI: trap_qualifier 0x%x trap_id 0x%x\n", 2991 packet_info->internal.trap_qualifier, 2992 packet_info->internal.trap_id)); 2993 } 2994 2995 /* Move forward to end of FHEI Trap extension */ 2996 if (fhei_size) { 2997 pkt_offset += 3; 2998 } 2999 3000 /* EEI extension */ 3001 if (eei_extension_present) { 3002 pkt_offset += BKN_DPP_INTERNAL_EXPLICIT_EDITING_INFOMATION_EXTENSION_SIZE_BYTE; 3003 } 3004 /* Learn extension */ 3005 if (learn_extension_present) { 3006 pkt_offset += BKN_DPP_INTERNAL_LEARN_EXTENSION_SIZE_BYTE; 3007 } 3008 3009 packet_info->system_header_size = pkt_offset; 3010 return; 3011 } 3012 3013 static int 3014 bkn_dpp_packet_header_parse( 3015 bkn_switch_info_t *sinfo, 3016 uint8_t *buff, 3017 uint32_t buff_len, 3018 bkn_dune_system_header_info_t *packet_info) 3019 { 3020 uint8_t is_inter_hdr_en = FALSE; 3021 uint8_t is_tsh_en = FALSE; 3022 uint8_t is_oamp_punted = FALSE; 3023 uint8_t is_trapped = FALSE; 3024 uint8_t is_oam_dm_tod_en = FALSE; 3025 uint8_t is_skip_udh = FALSE; 3026 3027 if ((sinfo == NULL) || (buff == NULL) || (packet_info == NULL)) { 3028 return -1; 3029 } 3030 3031 /* FTMH */ 3032 bkn_dpp_packet_parse_ftmh(sinfo, buff, buff_len, packet_info, &is_tsh_en, &is_inter_hdr_en); 3033 /* Check if packet is punted from OAMP */ 3034 if (sinfo->oamp_punt && (packet_info->ftmh.source_sys_port_aggregate == sinfo->oamp_punt)) { 3035 is_oamp_punted = TRUE; 3036 } 3037 /* OTSH */ 3038 if (is_tsh_en == TRUE) 3039 { 3040 bkn_dpp_packet_parse_otsh(sinfo, buff, buff_len, packet_info, &is_oam_dm_tod_en, &is_skip_udh); 3041 } 3042 /* Internal header is forced to be present if packet was punted to CPU by OAMP */ 3043 if (sinfo->oamp_punt && (packet_info->ftmh.source_sys_port_aggregate == sinfo->oamp_punt)) 3044 { 3045 is_inter_hdr_en = TRUE; 3046 } 3047 /* Internal */ 3048 if (is_inter_hdr_en) 3049 { 3050 bkn_dpp_packet_parse_internal(sinfo, buff, buff_len, packet_info, is_oamp_punted, &is_trapped); 3051 } 3052 /* Skip UDH for the outer layer when packet is punted by OAM for JR2 in JR1 mode */ 3053 if (device_is_dnx(sinfo) && is_oamp_punted) { 3054 is_skip_udh = TRUE; 3055 } 3056 /* UDH */ 3057 if (sinfo->udh_size && !is_skip_udh) { 3058 packet_info->system_header_size += sinfo->udh_size; 3059 } 3060 /* OAM DM TOD header */ 3061 if(is_oam_dm_tod_en) { 3062 packet_info->system_header_size += BKN_DPP_OAM_DM_TOD_SIZE_BYTE; 3063 } 3064 3065 /* Additional layer of system headers */ 3066 if (is_oamp_punted && is_trapped) 3067 { 3068 is_inter_hdr_en = FALSE; 3069 is_tsh_en = FALSE; 3070 is_oamp_punted = FALSE; 3071 is_trapped = FALSE; 3072 is_oam_dm_tod_en = FALSE; 3073 is_skip_udh = FALSE; 3074 3075 /* FTMH */ 3076 bkn_dpp_packet_parse_ftmh(sinfo, buff, buff_len, packet_info, &is_tsh_en, &is_inter_hdr_en); 3077 /* OTSH */ 3078 if (is_tsh_en == TRUE) 3079 { 3080 bkn_dpp_packet_parse_otsh(sinfo, buff, buff_len, packet_info, &is_oam_dm_tod_en, &is_skip_udh); 3081 } 3082 /* Internal */ 3083 if (is_inter_hdr_en) 3084 { 3085 bkn_dpp_packet_parse_internal(sinfo, buff, buff_len, packet_info, is_oamp_punted, &is_trapped); 3086 } 3087 /* OAMP Punted packets do not have UDH in the inner header for both JR1 and JR2 in JR1 mode */ 3088 /* OAM DM TOD header */ 3089 if(is_oam_dm_tod_en) { 3090 packet_info->system_header_size += BKN_DPP_OAM_DM_TOD_SIZE_BYTE; 3091 } 3092 } 3093 3094 DBG_DUNE(("Total length of headers is %u\n", packet_info->system_header_size)); 3095 3096 return 0; 3097 } 3098 3099 3100 static int 3101 bkn_dnx_packet_parse_ftmh( 3102 bkn_switch_info_t *sinfo, 3103 uint8_t *buf, 3104 uint32_t buf_len, 3105 bkn_dune_system_header_info_t *packet_info, 3106 uint8_t *is_tsh_en, 3107 uint8_t *is_inter_hdr_en) 3108 { 3109 uint32_t fld_val; 3110 uint32_t pkt_offset = packet_info->system_header_size; 3111 uint8_t tm_dst_ext_present = 0; 3112 uint8_t app_specific_ext_size = 0; 3113 uint8_t flow_id_ext_size = 0; 3114 uint8_t bier_bfr_ext_size = 0; 3115 3116 if ((sinfo == NULL) || (buf == NULL) || (packet_info == NULL)) { 3117 return -1; 3118 } 3119 3120 /* FTMH: Source-System-Port-Aggregate */ 3121 bkn_bitstream_get_field( 3122 &buf[pkt_offset], 3123 BKN_DNX_FTMH_SRC_SYS_PORT_AGGREGATE_MSB, 3124 BKN_DNX_FTMH_SRC_SYS_PORT_AGGREGATE_NOF_BITS, 3125 &fld_val); 3126 packet_info->ftmh.source_sys_port_aggregate = fld_val; 3127 /* FTMH: Action-Type */ 3128 bkn_bitstream_get_field( 3129 &buf[pkt_offset], 3130 BKN_DNX_FTMH_ACTION_TYPE_MSB, 3131 BKN_DNX_FTMH_ACTION_TYPE_NOF_BITS, 3132 &fld_val); 3133 packet_info->ftmh.action_type = fld_val; 3134 /* FTMH: PPH-Type TSH */ 3135 bkn_bitstream_get_field( 3136 &buf[pkt_offset], 3137 BKN_DNX_FTMH_PPH_TYPE_IS_TSH_EN_MSB, 3138 BKN_DNX_FTMH_PPH_TYPE_IS_TSH_EN_NOF_BITS, 3139 &fld_val); 3140 *is_tsh_en = fld_val; 3141 /* FTMH: PPH-Type PPH base */ 3142 bkn_bitstream_get_field( 3143 &buf[pkt_offset], 3144 BKN_DNX_FTMH_PPH_TYPE_IS_PPH_EN_MSB, 3145 BKN_DNX_FTMH_PPH_TYPE_IS_PPH_EN_NOF_BITS, 3146 &fld_val); 3147 *is_inter_hdr_en = fld_val; 3148 /* FTMH: TM-Destination-Extension-Present */ 3149 bkn_bitstream_get_field( 3150 &buf[pkt_offset], 3151 BKN_DNX_FTMH_TM_DST_EXT_PRESENT_MSB, 3152 BKN_DNX_FTMH_TM_DST_EXT_PRESENT_NOF_BITS, 3153 &fld_val); 3154 tm_dst_ext_present = fld_val; 3155 /* FTMH: Application-Specific-Extension-Size */ 3156 bkn_bitstream_get_field( 3157 &buf[pkt_offset], 3158 BKN_DNX_FTMH_APP_SPECIFIC_EXT_SIZE_MSB, 3159 BKN_DNX_FTMH_APP_SPECIFIC_EXT_SIZE_NOF_BITS, 3160 &fld_val); 3161 app_specific_ext_size = fld_val; 3162 /* FTMH: Flow-ID-Extension-Size */ 3163 bkn_bitstream_get_field( 3164 &buf[pkt_offset], 3165 BKN_DNX_FTMH_FLOW_ID_EXT_SIZE_MSB, 3166 BKN_DNX_FTMH_FLOW_ID_EXT_SIZE_NOF_BITS, 3167 &fld_val); 3168 flow_id_ext_size = fld_val; 3169 /* FTMH: BIER-BFR-Extension-Size */ 3170 bkn_bitstream_get_field( 3171 &buf[pkt_offset], 3172 BKN_DNX_FTMH_BIER_BFR_EXT_SIZE_MSB, 3173 BKN_DNX_FTMH_BIER_BFR_EXT_SIZE_NOF_BITS, 3174 &fld_val); 3175 bier_bfr_ext_size = fld_val; 3176 3177 pkt_offset += BKN_DNX_FTMH_BASE_SIZE; 3178 3179 DBG_DUNE(("FTMH(10-%u): source-system-port 0x%x action_type %u is_tsh_en %u is_inter_hdr_en %u\n", 3180 pkt_offset, packet_info->ftmh.source_sys_port_aggregate, 3181 packet_info->ftmh.action_type, *is_tsh_en, *is_inter_hdr_en)); 3182 3183 /* FTMH LB-Key Extension */ 3184 if (sinfo->ftmh_lb_key_ext_size > 0) 3185 { 3186 pkt_offset += sinfo->ftmh_lb_key_ext_size; 3187 DBG_DUNE(("FTMH LB-Key Extension(%u-%u) is present\n", sinfo->ftmh_lb_key_ext_size, pkt_offset)); 3188 } 3189 /* FTMH Stacking Extension */ 3190 if (sinfo->ftmh_stacking_ext_size > 0) 3191 { 3192 pkt_offset += sinfo->ftmh_stacking_ext_size; 3193 DBG_DUNE(("FTMH Stacking Extension(%u-%u) is present\n", sinfo->ftmh_stacking_ext_size, pkt_offset)); 3194 } 3195 /* FTMH BIER BFR Extension */ 3196 if (bier_bfr_ext_size > 0) 3197 { 3198 pkt_offset += BKN_DNX_FTMH_BIER_BFR_EXT_SIZE; 3199 DBG_DUNE(("FTMH BIER BFR Extension(2-%u) is present\n", pkt_offset)); 3200 } 3201 /* FTMH TM Destination Extension */ 3202 if (tm_dst_ext_present > 0) 3203 { 3204 pkt_offset += BKN_DNX_FTMH_TM_DST_EXT_SIZE; 3205 DBG_DUNE(("FTMH TM Destination Extension(3-%u) is present\n", pkt_offset)); 3206 } 3207 /* FTMH Application Specific Extension */ 3208 if (app_specific_ext_size > 0) 3209 { 3210 pkt_offset += BKN_DNX_FTMH_APP_SPECIFIC_EXT_SIZE; 3211 DBG_DUNE(("FTMH Application Specific Extension(6-%u) is present\n", pkt_offset)); 3212 } 3213 /* FTMH Flow-ID Extension */ 3214 if (flow_id_ext_size > 0) 3215 { 3216 pkt_offset += BKN_DNX_FTMH_FLOW_ID_EXT_SIZE; 3217 DBG_DUNE(("FTMH Flow-ID Extension(3-%u) is present\n", pkt_offset)); 3218 } 3219 3220 packet_info->system_header_size = pkt_offset; 3221 3222 return 0; 3223 } 3224 3225 3226 static int 3227 bkn_dnx_packet_parse_internal( 3228 bkn_switch_info_t *sinfo, 3229 uint8_t *buf, 3230 uint32_t buf_len, 3231 bkn_dune_system_header_info_t *packet_info, 3232 uint8_t is_oamp_punted, 3233 uint8_t *is_trapped) 3234 { 3235 uint32_t fld_val; 3236 uint32_t pkt_offset = packet_info->system_header_size; 3237 uint8_t learn_ext_present; 3238 uint8_t fhei_size; 3239 uint8_t lif_ext_type; 3240 uint8_t udh_en = sinfo->udh_enable; 3241 3242 if ((sinfo == NULL) || (buf == NULL) || (packet_info == NULL)) { 3243 return -1; 3244 } 3245 3246 /* Internal: Forward-Domain */ 3247 bkn_bitstream_get_field( 3248 &buf[pkt_offset], 3249 BKN_DNX_INTERNAL_12_FORWARD_DOMAIN_MSB, 3250 BKN_DNX_INTERNAL_12_FORWARD_DOMAIN_NOF_BITS, 3251 &fld_val); 3252 packet_info->internal.forward_domain = fld_val; 3253 /* Internal: Learn-Extension-Present */ 3254 bkn_bitstream_get_field( 3255 &buf[pkt_offset], 3256 BKN_DNX_INTERNAL_12_LEARN_EXT_PRESENT_MSB, 3257 BKN_DNX_INTERNAL_12_LEARN_EXT_PRESENT_NOF_BITS, 3258 &fld_val); 3259 learn_ext_present = fld_val; 3260 /* Internal: FHEI-Size */ 3261 bkn_bitstream_get_field( 3262 &buf[pkt_offset], 3263 BKN_DNX_INTERNAL_12_FHEI_SIZE_MSB, 3264 BKN_DNX_INTERNAL_12_FHEI_SIZE_NOF_BITS, 3265 &fld_val); 3266 fhei_size = fld_val; 3267 /* Internal: LIF-Extension-Type */ 3268 bkn_bitstream_get_field( 3269 &buf[pkt_offset], 3270 BKN_DNX_INTERNAL_12_LIF_EXT_TYPE_MSB, 3271 BKN_DNX_INTERNAL_12_LIF_EXT_TYPE_NOF_BITS, 3272 &fld_val); 3273 lif_ext_type = fld_val; 3274 3275 pkt_offset += BKN_DNX_INTERNAL_BASE_TYPE_12; 3276 DBG_DUNE(("Internal(12-%u): FWD_DOMAIN %d, LEARN_EXT %d, FHEI_SIZE %d, LIF_EXT %d \n", 3277 pkt_offset, packet_info->internal.forward_domain, 3278 learn_ext_present, fhei_size, lif_ext_type)); 3279 3280 if (fhei_size) 3281 { 3282 switch (fhei_size) 3283 { 3284 case BKN_DNX_INTERNAL_FHEI_TYPE_SZ0: 3285 pkt_offset += BKN_DNX_INTERNAL_FHEI_SZ0_SIZE; 3286 DBG_DUNE(("FHEI(3-%u) is present\n", pkt_offset)); 3287 break; 3288 case BKN_DNX_INTERNAL_FHEI_TYPE_SZ1: 3289 /* FHEI: Type */ 3290 bkn_bitstream_get_field( 3291 &buf[pkt_offset], 3292 BKN_DNX_INTERNAL_FHEI_TRAP_5B_TYPE_MSB, 3293 BKN_DNX_INTERNAL_FHEI_TRAP_5B_TYPE_NOF_BITS, 3294 &fld_val); 3295 /* FHEI-Size == 5B, FHEI-Type == Trap/Sniff */ 3296 if (fld_val == 0x5) 3297 { 3298 *is_trapped = TRUE; 3299 /* FHEI: Qualifier */ 3300 bkn_bitstream_get_field( 3301 &buf[pkt_offset], 3302 BKN_DNX_INTERNAL_FHEI_TRAP_5B_QUALIFIER_MSB, 3303 BKN_DNX_INTERNAL_FHEI_TRAP_5B_QUALIFIER_NOF_BITS, 3304 &fld_val); 3305 packet_info->internal.trap_qualifier = fld_val; 3306 /* FHEI: Code */ 3307 bkn_bitstream_get_field( 3308 &buf[pkt_offset], 3309 BKN_DNX_INTERNAL_FHEI_TRAP_5B_CODE_MSB, 3310 BKN_DNX_INTERNAL_FHEI_TRAP_5B_CODE_NOF_BITS, 3311 &fld_val); 3312 packet_info->internal.trap_id= fld_val; 3313 } 3314 pkt_offset += BKN_DNX_INTERNAL_FHEI_SZ1_SIZE; 3315 DBG_DUNE(("FHEI(5-%u): code 0x%x qualifier 0x%x\n", pkt_offset, packet_info->internal.trap_id, packet_info->internal.trap_qualifier)); 3316 break; 3317 case BKN_DNX_INTERNAL_FHEI_TYPE_SZ2: 3318 pkt_offset += BKN_DNX_INTERNAL_FHEI_SZ1_SIZE; 3319 DBG_DUNE(("FHEI(8-%u) is present\n", pkt_offset)); 3320 break; 3321 } 3322 } 3323 3324 /* PPH LIF Extension */ 3325 if (lif_ext_type) 3326 { 3327 pkt_offset += sinfo->pph_lif_ext_size[lif_ext_type]; 3328 DBG_DUNE(("PPH LIF Extension(%d-%u) is present\n", sinfo->pph_lif_ext_size[lif_ext_type], pkt_offset)); 3329 } 3330 3331 /* PPH Learn Extension */ 3332 if (learn_ext_present) 3333 { 3334 pkt_offset += BKN_DNX_INTERNAL_LEARN_EXT_SIZE; 3335 DBG_DUNE(("PPH Learn Extension(19-%u) is present\n", pkt_offset)); 3336 } 3337 3338 /** Skip UDH If packet is punted to CPU by OAMP */ 3339 if (is_oamp_punted) { 3340 udh_en = FALSE; 3341 } 3342 3343 /* UDH Header */ 3344 if (udh_en) 3345 { 3346 uint8_t data_type_0; 3347 uint8_t data_type_1; 3348 uint8_t data_type_2; 3349 uint8_t data_type_3; 3350 3351 /* UDH: UDH-Data-Type[0] */ 3352 bkn_bitstream_get_field( 3353 &buf[pkt_offset], 3354 BKN_DNX_UDH_DATA_TYPE_0_MSB, 3355 BKN_DNX_UDH_DATA_TYPE_0_NOF_BITS, 3356 &fld_val); 3357 data_type_0 = fld_val; 3358 /* UDH: UDH-Data-Type[1] */ 3359 bkn_bitstream_get_field( 3360 &buf[pkt_offset], 3361 BKN_DNX_UDH_DATA_TYPE_1_MSB, 3362 BKN_DNX_UDH_DATA_TYPE_1_NOF_BITS, 3363 &fld_val); 3364 data_type_1 = fld_val; 3365 /* UDH: UDH-Data-Type[2] */ 3366 bkn_bitstream_get_field( 3367 &buf[pkt_offset], 3368 BKN_DNX_UDH_DATA_TYPE_2_MSB, 3369 BKN_DNX_UDH_DATA_TYPE_2_NOF_BITS, 3370 &fld_val); 3371 data_type_2 = fld_val; 3372 /* UDH: UDH-Data-Type[3] */ 3373 bkn_bitstream_get_field( 3374 &buf[pkt_offset], 3375 BKN_DNX_UDH_DATA_TYPE_3_MSB, 3376 BKN_DNX_UDH_DATA_TYPE_3_NOF_BITS, 3377 &fld_val); 3378 data_type_3 = fld_val; 3379 pkt_offset += BKN_DNX_UDH_BASE_SIZE; 3380 pkt_offset += sinfo->udh_length_type[data_type_0]; 3381 pkt_offset += sinfo->udh_length_type[data_type_1]; 3382 pkt_offset += sinfo->udh_length_type[data_type_2]; 3383 pkt_offset += sinfo->udh_length_type[data_type_3]; 3384 DBG_DUNE(("UDH base(1-%u) is present\n", pkt_offset)); 3385 } 3386 3387 packet_info->system_header_size = pkt_offset; 3388 3389 return 0; 3390 } 3391 3392 static int 3393 bkn_dnx_packet_header_parse( 3394 bkn_switch_info_t *sinfo, 3395 uint8_t *buff, 3396 uint32_t buff_len, 3397 bkn_dune_system_header_info_t *packet_info) 3398 { 3399 uint8_t is_inter_hdr_en = FALSE; 3400 uint8_t is_tsh_en = FALSE; 3401 uint8_t is_oamp_punted = FALSE; 3402 uint8_t is_trapped = FALSE; 3403 3404 if ((sinfo == NULL) || (buff == NULL) || (packet_info == NULL)) { 3405 return -1; 3406 } 3407 3408 /* FTMH */ 3409 bkn_dnx_packet_parse_ftmh(sinfo, buff, buff_len, packet_info, &is_tsh_en, &is_inter_hdr_en); 3410 3411 /* Time-Stamp */ 3412 if (is_tsh_en == TRUE) 3413 { 3414 packet_info->system_header_size += BKN_DNX_TSH_SIZE; 3415 DBG_DUNE(("Time-Stamp Header(4-%u) is present\n", packet_info->system_header_size)); 3416 } 3417 3418 /* Check if packet was punted to CPU by OAMP */ 3419 if ((packet_info->ftmh.source_sys_port_aggregate == 232) 3420 || (packet_info->ftmh.source_sys_port_aggregate == 233)) 3421 { 3422 is_oamp_punted = TRUE; 3423 } 3424 3425 /* Internal */ 3426 if (is_inter_hdr_en) 3427 { 3428 bkn_dnx_packet_parse_internal(sinfo, buff, buff_len, packet_info, is_oamp_punted, &is_trapped); 3429 } 3430 3431 if (is_oamp_punted) 3432 { 3433 is_inter_hdr_en = FALSE; 3434 is_tsh_en = FALSE; 3435 is_oamp_punted = FALSE; 3436 is_trapped = FALSE; 3437 3438 /* FTMH */ 3439 bkn_dnx_packet_parse_ftmh(sinfo, buff, buff_len, packet_info, &is_tsh_en, &is_inter_hdr_en); 3440 /* Time-Stamp */ 3441 if (is_tsh_en == TRUE) 3442 { 3443 packet_info->system_header_size += BKN_DNX_TSH_SIZE; 3444 DBG_DUNE(("Time-Stamp Header(4-%u) is present\n", packet_info->system_header_size)); 3445 } 3446 /* Internal */ 3447 if (is_inter_hdr_en) 3448 { 3449 bkn_dnx_packet_parse_internal(sinfo, buff, buff_len, packet_info, is_oamp_punted, &is_trapped); 3450 } 3451 } 3452 3453 DBG_DUNE(("Total length of headers is %u\n", packet_info->system_header_size)); 3454 3455 return 0; 3456 } 3457 3458 static int 3459 bkn_packet_header_parse(bkn_switch_info_t *sinfo, uint8_t *buf, uint32_t buf_len, bkn_dune_system_header_info_t *packet_info) 3460 { 3461 if (device_is_dpp(sinfo)) 3462 { 3463 bkn_dpp_packet_header_parse(sinfo, buf, buf_len, packet_info); 3464 } 3465 else if (device_is_dnx(sinfo)) 3466 { 3467 if (sinfo->system_headers_mode == BKN_DNX_JR2_MODE) 3468 { 3469 /* Jericho 2 mode */ 3470 bkn_dnx_packet_header_parse(sinfo, buf, buf_len, packet_info); 3471 } 3472 else 3473 { 3474 /* Jericho/QMX/QAX mode */ 3475 bkn_dpp_packet_header_parse(sinfo, buf, buf_len, packet_info); 3476 } 3477 } 3478 return 0; 3479 } 3480 3481 static int 3482 bkn_do_api_rx(bkn_switch_info_t *sinfo, int chan, int budget) 3483 { 3484 bkn_priv_t *priv; 3485 bkn_dcb_chain_t *dcb_chain; 3486 struct sk_buff *skb; 3487 bkn_filter_t cbf; 3488 bkn_filter_t *filter; 3489 uint32_t err_woff; 3490 uint32_t *dcb, *meta; 3491 uint8_t *pkt; 3492 uint64_t pkt_dma; 3493 int drop_api; 3494 int ethertype; 3495 int pktlen; 3496 int idx; 3497 int dcbs_done = 0; 3498 bkn_dune_system_header_info_t packet_info; 3499 uint32_t sand_scratch_data[BKN_SAND_SCRATCH_DATA_SIZE] = {0}; 3500 3501 dcb_chain = sinfo->rx[chan].api_dcb_chain; 3502 if (dcb_chain == NULL) { 3503 /* No active chains */ 3504 return 0; 3505 } 3506 3507 while (dcb_chain->dcb_cur < dcb_chain->dcb_cnt) { 3508 dcb = &dcb_chain->dcb_mem[dcb_chain->dcb_cur * sinfo->dcb_wsize]; 3509 DBG_VERB(("DCB %2d: 0x%08x\n", 3510 dcb_chain->dcb_cur, dcb[sinfo->dcb_wsize-1])); 3511 if ((dcb[sinfo->dcb_wsize-1] & (1 << 31)) == 0) { 3512 break; 3513 } 3514 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 3515 /* Handle for Continuous DMA mode */ 3516 if (dcbs_done >= budget) { 3517 break; 3518 } 3519 if ((sinfo->cmic_type == 'x' && dcb[2] & (1 << 18)) || 3520 (sinfo->cmic_type != 'x' && dcb[1] & (1 << 18))) { 3521 dcb[sinfo->dcb_wsize-1] |= SOC_DCB_KNET_DONE; 3522 bkn_api_rx_chain_done(sinfo, chan); 3523 dcb_chain = sinfo->rx[chan].api_dcb_chain; 3524 if (dcb_chain == NULL) { 3525 break; 3526 } 3527 continue; 3528 } 3529 } 3530 if ((sinfo->cmic_type == 'x' && (dcb[2] & (1 << 16)) == 0) || 3531 (sinfo->cmic_type != 'x' && (dcb[1] & (1 << 16)) == 0)) { 3532 sinfo->rx[chan].chain_complete = 1; 3533 } 3534 sinfo->rx[chan].pkts++; 3535 if (sinfo->cmic_type == 'x') { 3536 pkt_dma = BUS_TO_DMA_HI(dcb[1]); 3537 pkt_dma = pkt_dma << 32 | dcb[0]; 3538 } else { 3539 pkt_dma = dcb[0]; 3540 } 3541 pkt = (uint8_t *)kernel_bde->p2l(sinfo->dev_no, (sal_paddr_t)pkt_dma); 3542 3543 if (device_is_sand(sinfo)) { 3544 err_woff = BKN_SAND_SCRATCH_DATA_SIZE - 1; 3545 sand_scratch_data[err_woff] = dcb[sinfo->dcb_wsize-1]; 3546 meta = (uint32_t *)pkt; 3547 } else { 3548 if (sinfo->cmic_type == 'x') { 3549 meta = (uint32_t *)pkt; 3550 err_woff = sinfo->pkt_hdr_size / sizeof(uint32_t) - 1; 3551 meta[err_woff] = dcb[sinfo->dcb_wsize-1]; 3552 } else { 3553 meta = dcb; 3554 err_woff = sinfo->dcb_wsize - 1; 3555 } 3556 } 3557 3558 pktlen = dcb[sinfo->dcb_wsize-1] & SOC_DCB_KNET_COUNT_MASK; 3559 bkn_dump_pkt(pkt, pktlen, XGS_DMA_RX_CHAN); 3560 3561 if (device_is_sand(sinfo)) { 3562 memset(&packet_info, 0, sizeof(bkn_dune_system_header_info_t)); 3563 /* decode system headers and fill sratch data */ 3564 bkn_packet_header_parse(sinfo, pkt, (uint32_t)pktlen, &packet_info); 3565 bkn_bitstream_set_field(sand_scratch_data, 0, 16, packet_info.internal.trap_id); 3566 bkn_bitstream_set_field(sand_scratch_data, 16, 16, packet_info.internal.trap_qualifier); 3567 bkn_bitstream_set_field(sand_scratch_data, 32, 16, packet_info.ftmh.source_sys_port_aggregate); 3568 bkn_bitstream_set_field(sand_scratch_data, 48, 16, packet_info.internal.forward_domain); 3569 bkn_bitstream_set_field(sand_scratch_data, 64, 2, packet_info.ftmh.action_type); 3570 3571 if (force_tagged) { 3572 uint8_t *eth_hdr = pkt + packet_info.system_header_size; 3573 uint16_t tpid = 0; 3574 3575 tpid = PKT_U16_GET(eth_hdr, 12); 3576 if (packet_is_untagged(tpid)) { 3577 int raw_packet_len = pktlen - packet_info.system_header_size; 3578 uint32_t vid = 0; 3579 3580 if ((pktlen + 4) < rx_buffer_size) { 3581 for (idx = (raw_packet_len - 1); idx >= 12; idx--) { 3582 eth_hdr[idx+4] = eth_hdr[idx]; 3583 } 3584 if (ft_vid) { 3585 vid = ft_vid; 3586 } 3587 else if (packet_info.internal.forward_domain) { 3588 vid = packet_info.internal.forward_domain & 0xfff; 3589 } 3590 else { 3591 vid = 1; 3592 } 3593 DBG_DUNE(("add vlan tag (%d) to untagged packets\n", vid)); 3594 3595 eth_hdr[12] = (ft_tpid >> 8) & 0xff; 3596 eth_hdr[13] = ft_tpid & 0xff; 3597 eth_hdr[14] = (((ft_pri & 0x7) << 5) | ((ft_cfi & 0x1) << 4) | ((vid >> 8) & 0xf)) & 0xff; 3598 eth_hdr[15] = vid & 0xff; 3599 /* reset packet length in DCB */ 3600 pktlen += 4; 3601 bkn_dump_pkt(pkt, pktlen, XGS_DMA_RX_CHAN); 3602 dcb[sinfo->dcb_wsize-1] &= ~SOC_DCB_KNET_COUNT_MASK; 3603 dcb[sinfo->dcb_wsize-1] |= pktlen & SOC_DCB_KNET_COUNT_MASK; 3604 } 3605 } 3606 } 3607 } 3608 3609 if (device_is_sand(sinfo)) { 3610 filter = bkn_match_rx_pkt(sinfo, pkt + packet_info.system_header_size, 3611 pktlen - packet_info.system_header_size, sand_scratch_data, chan, &cbf); 3612 } else { 3613 filter = bkn_match_rx_pkt(sinfo, pkt + sinfo->pkt_hdr_size, 3614 pktlen - sinfo->pkt_hdr_size, meta, chan, &cbf); 3615 } 3616 if ((dcb[sinfo->dcb_wsize-1] & 0xf0000) != 0x30000) { 3617 /* Fragment or error */ 3618 if (filter && filter->kf.mask.w[err_woff] == 0) { 3619 /* Drop unless DCB status is part of filter */ 3620 filter = NULL; 3621 } 3622 } 3623 drop_api = 1; 3624 if (filter) { 3625 DBG_FLTR(("Match filter ID %d\n", filter->kf.id)); 3626 switch (filter->kf.dest_type) { 3627 case KCOM_DEST_T_API: 3628 DBG_FLTR(("Send to Rx API\n")); 3629 sinfo->rx[chan].pkts_f_api++; 3630 drop_api = 0; 3631 break; 3632 case KCOM_DEST_T_NETIF: 3633 priv = bkn_netif_lookup(sinfo, filter->kf.dest_id); 3634 if (priv) { 3635 /* Check that software link is up */ 3636 if (!netif_carrier_ok(priv->dev)) { 3637 sinfo->rx[chan].pkts_d_no_link++; 3638 break; 3639 } 3640 3641 if (device_is_sand(sinfo)) { 3642 pkt += packet_info.system_header_size; 3643 pktlen -= packet_info.system_header_size; 3644 } else if (sinfo->cmic_type == 'x') { 3645 pkt += sinfo->pkt_hdr_size; 3646 pktlen -= sinfo->pkt_hdr_size; 3647 } 3648 3649 /* Add 2 bytes for IP header alignment (see below) */ 3650 skb = dev_alloc_skb(pktlen + RCPU_RX_ENCAP_SIZE + 2); 3651 if (skb == NULL) { 3652 sinfo->rx[chan].pkts_d_no_skb++; 3653 break; 3654 } 3655 skb_reserve(skb, RCPU_RX_ENCAP_SIZE); 3656 3657 DBG_FLTR(("Send to netif %d (%s)\n", 3658 priv->id, priv->dev->name)); 3659 sinfo->rx[chan].pkts_f_netif++; 3660 skb->dev = priv->dev; 3661 skb_reserve(skb, 2); /* 16 byte align the IP fields. */ 3662 3663 /* Save for RCPU before stripping tag */ 3664 ethertype = PKT_U16_GET(pkt, 16); 3665 if ((priv->flags & KCOM_NETIF_F_KEEP_RX_TAG) == 0) { 3666 uint16_t vlan_proto = PKT_U16_GET(pkt, 12); 3667 3668 if (filter->kf.flags & KCOM_FILTER_F_STRIP_TAG) { 3669 /* Strip the VLAN tag */ 3670 if (vlan_proto == ETH_P_8021Q || 3671 vlan_proto == ETH_P_8021AD) { 3672 DBG_FLTR(("Strip VLAN tag\n")); 3673 for (idx = 11; idx >= 0; idx--) { 3674 pkt[idx+4] = pkt[idx]; 3675 } 3676 pktlen -= 4; 3677 pkt += 4; 3678 } 3679 } else { 3680 /* 3681 * Mark packet as VLAN-tagged, otherwise newer 3682 * kernels will strip the tag. 3683 */ 3684 uint16_t tci = PKT_U16_GET(pkt, 14); 3685 3686 if (priv->flags & KCOM_NETIF_F_RCPU_ENCAP) { 3687 bkn_vlan_hwaccel_put_tag(skb, ETH_P_8021Q, tci); 3688 } else { 3689 if (vlan_proto == ETH_P_8021AD) { 3690 bkn_vlan_hwaccel_put_tag 3691 (skb, ETH_P_8021AD, tci); 3692 } else { 3693 bkn_vlan_hwaccel_put_tag 3694 (skb, ETH_P_8021Q, tci); 3695 } 3696 } 3697 } 3698 } 3699 3700 skb_copy_to_linear_data(skb, pkt, pktlen); 3701 if (device_is_sand(sinfo)) { 3702 /* CRC has been stripped */ 3703 skb_put(skb, pktlen); 3704 } else { 3705 skb_put(skb, pktlen - 4); /* Strip CRC */ 3706 } 3707 priv->stats.rx_packets++; 3708 priv->stats.rx_bytes += skb->len; 3709 3710 /* Optional SKB updates */ 3711 KNET_SKB_CB(skb)->dcb_type = sinfo->dcb_type & 0xFFFF; 3712 if (knet_rx_cb != NULL) { 3713 KNET_SKB_CB(skb)->netif_user_data = priv->cb_user_data; 3714 KNET_SKB_CB(skb)->filter_user_data = filter->kf.cb_user_data; 3715 if (device_is_sand(sinfo)) { 3716 skb = knet_rx_cb(skb, sinfo->dev_no, sand_scratch_data); 3717 } 3718 else { 3719 skb = knet_rx_cb(skb, sinfo->dev_no, meta); 3720 } 3721 if (skb == NULL) { 3722 /* Consumed by call-back */ 3723 sinfo->rx[chan].pkts_d_callback++; 3724 break; 3725 } 3726 } 3727 3728 /* Do Rx timestamping */ 3729 if (priv->rx_hwts) { 3730 bkn_hw_tstamp_rx_set(sinfo, priv->phys_port, skb, meta); 3731 } 3732 3733 if (priv->flags & KCOM_NETIF_F_RCPU_ENCAP) { 3734 bkn_add_rcpu_encap(sinfo, skb, meta, packet_info.system_header_size); 3735 DBG_PDMP(("After add RCPU ENCAP\n")); 3736 bkn_dump_pkt(skb->data, pktlen + RCPU_RX_ENCAP_SIZE, XGS_DMA_RX_CHAN); 3737 } 3738 skb->protocol = eth_type_trans(skb, skb->dev); 3739 if (filter->kf.dest_proto) { 3740 skb->protocol = filter->kf.dest_proto; 3741 } 3742 if (priv->flags & KCOM_NETIF_F_RCPU_ENCAP) { 3743 bkn_eth_type_update(skb, ethertype); 3744 } 3745 DBG_DUNE(("skb protocol 0x%04x\n",skb->protocol)); 3746 3747 /* Unlock while calling up network stack */ 3748 spin_unlock(&sinfo->lock); 3749 if (use_napi) { 3750 netif_receive_skb(skb); 3751 } else { 3752 netif_rx(skb); 3753 } 3754 spin_lock(&sinfo->lock); 3755 3756 if (filter->kf.mirror_type == KCOM_DEST_T_API || 3757 dbg_pkt_enable) { 3758 DBG_FLTR(("Mirror to Rx API\n")); 3759 sinfo->rx[chan].pkts_m_api++; 3760 drop_api = 0; 3761 } 3762 } else { 3763 DBG_FLTR(("Unknown netif %d\n", 3764 filter->kf.dest_id)); 3765 sinfo->rx[chan].pkts_d_unkn_netif++; 3766 } 3767 break; 3768 default: 3769 /* Drop packet */ 3770 DBG_FLTR(("Unknown dest type %d\n", 3771 filter->kf.dest_type)); 3772 sinfo->rx[chan].pkts_d_unkn_dest++; 3773 break; 3774 } 3775 } else { 3776 DBG_PKT(("Rx packet dropped.\n")); 3777 sinfo->rx[chan].pkts_d_no_match++; 3778 } 3779 if (drop_api) { 3780 /* If count is zero, the DCB will just be recycled */ 3781 dcb[sinfo->dcb_wsize-1] &= ~SOC_DCB_KNET_COUNT_MASK; 3782 } 3783 dcb[sinfo->dcb_wsize-1] |= SOC_DCB_KNET_DONE; 3784 dcb_chain->dcb_cur++; 3785 dcbs_done++; 3786 } 3787 3788 return dcbs_done; 3789 } 3790 3791 static int 3792 bkn_do_skb_rx(bkn_switch_info_t *sinfo, int chan, int budget) 3793 { 3794 bkn_priv_t *priv; 3795 bkn_desc_info_t *desc; 3796 struct sk_buff *skb; 3797 bkn_filter_t cbf; 3798 bkn_filter_t *filter; 3799 uint32_t err_woff; 3800 uint32_t *dcb, *meta; 3801 int ethertype; 3802 int pktlen; 3803 int idx; 3804 int dcbs_done = 0; 3805 bkn_dune_system_header_info_t packet_info = {0}; 3806 uint32_t sand_scratch_data[BKN_SAND_SCRATCH_DATA_SIZE] = {0}; 3807 uint8_t sand_system_headers[RCPU_RX_META_SIZE] = {0}; 3808 uint8_t *pkt = NULL; 3809 3810 if (!sinfo->rx[chan].running) { 3811 /* Rx not ready */ 3812 return 0; 3813 } 3814 3815 while (dcbs_done < budget) { 3816 char str[32]; 3817 sprintf(str, "Rx DCB (%d)", sinfo->rx[chan].dirty); 3818 desc = &sinfo->rx[chan].desc[sinfo->rx[chan].dirty]; 3819 dcb = desc->dcb_mem; 3820 bkn_dump_dcb(str, dcb, sinfo->dcb_wsize, XGS_DMA_RX_CHAN); 3821 if ((dcb[sinfo->dcb_wsize-1] & (1 << 31)) == 0) { 3822 break; 3823 } 3824 if ((sinfo->cmic_type == 'x' && (dcb[2] & (1 << 16)) == 0) || 3825 (sinfo->cmic_type != 'x' && (dcb[1] & (1 << 16)) == 0)) { 3826 sinfo->rx[chan].chain_complete = 1; 3827 /* Request one extra poll to check for chain done interrupt */ 3828 if (sinfo->napi_poll_mode) { 3829 sinfo->napi_poll_again = 1; 3830 } 3831 } 3832 sinfo->rx[chan].pkts++; 3833 skb = desc->skb; 3834 3835 if (device_is_sand(sinfo)) { 3836 err_woff = BKN_SAND_SCRATCH_DATA_SIZE - 1; 3837 sand_scratch_data[err_woff] = dcb[sinfo->dcb_wsize-1]; 3838 meta = (uint32_t *)skb->data; 3839 pkt = skb->data; 3840 } else { 3841 if (sinfo->cmic_type == 'x') { 3842 meta = (uint32_t *)skb->data; 3843 err_woff = sinfo->pkt_hdr_size / sizeof(uint32_t) - 1; 3844 meta[err_woff] = dcb[sinfo->dcb_wsize-1]; 3845 } else { 3846 meta = dcb; 3847 err_woff = sinfo->dcb_wsize - 1; 3848 } 3849 } 3850 3851 pktlen = dcb[sinfo->dcb_wsize-1] & 0xffff; 3852 priv = netdev_priv(sinfo->dev); 3853 DBG_DCB_RX(("Rx%d SKB DMA done (%d).\n", chan, sinfo->rx[chan].dirty)); 3854 BKN_DMA_UNMAP_SINGLE(sinfo->dma_dev, 3855 desc->skb_dma, desc->dma_size, 3856 BKN_DMA_FROMDEV); 3857 desc->skb_dma = 0; 3858 bkn_dump_pkt(skb->data, pktlen, XGS_DMA_RX_CHAN); 3859 3860 if (device_is_sand(sinfo)) { 3861 memset(&packet_info, 0, sizeof(bkn_dune_system_header_info_t)); 3862 /* decode system headers and fill sratch data */ 3863 bkn_packet_header_parse(sinfo, pkt, (uint32_t)pktlen, &packet_info); 3864 bkn_bitstream_set_field(sand_scratch_data, 0, 16, packet_info.internal.trap_id); 3865 bkn_bitstream_set_field(sand_scratch_data, 16, 16, packet_info.internal.trap_qualifier); 3866 bkn_bitstream_set_field(sand_scratch_data, 32, 16, packet_info.ftmh.source_sys_port_aggregate); 3867 bkn_bitstream_set_field(sand_scratch_data, 48, 16, packet_info.internal.forward_domain); 3868 bkn_bitstream_set_field(sand_scratch_data, 64, 2, packet_info.ftmh.action_type); 3869 memcpy(sand_system_headers, pkt, 3870 ((packet_info.system_header_size > RCPU_RX_META_SIZE) ? RCPU_RX_META_SIZE : packet_info.system_header_size)); 3871 meta = (uint32_t *)sand_system_headers; 3872 if (force_tagged) { 3873 uint8_t *eth_hdr = pkt + packet_info.system_header_size; 3874 uint16_t tpid = 0; 3875 3876 tpid = PKT_U16_GET(eth_hdr, 12); 3877 if (packet_is_untagged(tpid)) { 3878 int raw_packet_len = pktlen - packet_info.system_header_size; 3879 uint32_t vid = 0; 3880 3881 if ((pktlen + 4) < rx_buffer_size) { 3882 for (idx = (raw_packet_len - 1); idx >= 12; idx--) { 3883 eth_hdr[idx+4] = eth_hdr[idx]; 3884 } 3885 if (ft_vid) { 3886 vid = ft_vid; 3887 } 3888 else if (packet_info.internal.forward_domain) { 3889 vid = packet_info.internal.forward_domain & 0xfff; 3890 } 3891 else { 3892 vid = 1; 3893 } 3894 DBG_DUNE(("add vlan tag (%d) to untagged packets\n", vid)); 3895 eth_hdr[12] = (ft_tpid >> 8) & 0xff; 3896 eth_hdr[13] = ft_tpid & 0xff; 3897 eth_hdr[14] = (((ft_pri & 0x7) << 5) | ((ft_cfi & 0x1) << 4) | ((vid >> 8) & 0xf)) & 0xff; 3898 eth_hdr[15] = vid & 0xff; 3899 /* reset packet length in DCB */ 3900 pktlen += 4; 3901 bkn_dump_pkt(pkt, pktlen, XGS_DMA_RX_CHAN); 3902 dcb[sinfo->dcb_wsize-1] &= ~SOC_DCB_KNET_COUNT_MASK; 3903 dcb[sinfo->dcb_wsize-1] |= pktlen & SOC_DCB_KNET_COUNT_MASK; 3904 } 3905 } 3906 } 3907 } 3908 if (device_is_sand(sinfo)) { 3909 filter = bkn_match_rx_pkt(sinfo, skb->data + packet_info.system_header_size, 3910 pktlen - packet_info.system_header_size, sand_scratch_data, chan, &cbf); 3911 } else { 3912 filter = bkn_match_rx_pkt(sinfo, skb->data + sinfo->pkt_hdr_size, 3913 pktlen - sinfo->pkt_hdr_size, meta, chan, &cbf); 3914 } 3915 if ((dcb[sinfo->dcb_wsize-1] & 0xf0000) != 0x30000) { 3916 /* Fragment or error */ 3917 priv->stats.rx_errors++; 3918 if (filter && filter->kf.mask.w[err_woff] == 0) { 3919 /* Drop unless DCB status is part of filter */ 3920 filter = NULL; 3921 } 3922 } 3923 DBG_PKT(("Rx packet (%d bytes).\n", pktlen)); 3924 if (filter) { 3925 DBG_FLTR(("Match filter ID %d\n", filter->kf.id)); 3926 switch (filter->kf.dest_type) { 3927 case KCOM_DEST_T_API: 3928 DBG_FLTR(("Send to Rx API\n")); 3929 sinfo->rx[chan].pkts_f_api++; 3930 bkn_api_rx_copy_from_skb(sinfo, chan, desc); 3931 break; 3932 case KCOM_DEST_T_NETIF: 3933 priv = bkn_netif_lookup(sinfo, filter->kf.dest_id); 3934 if (priv) { 3935 /* Check that software link is up */ 3936 if (!netif_carrier_ok(priv->dev)) { 3937 sinfo->rx[chan].pkts_d_no_link++; 3938 break; 3939 } 3940 DBG_FLTR(("Send to netif %d (%s)\n", 3941 priv->id, priv->dev->name)); 3942 sinfo->rx[chan].pkts_f_netif++; 3943 3944 if ((filter->kf.mirror_type == KCOM_DEST_T_API) || dbg_pkt_enable) { 3945 sinfo->rx[chan].pkts_m_api++; 3946 bkn_api_rx_copy_from_skb(sinfo, chan, desc); 3947 } 3948 3949 if (device_is_sand(sinfo)) { 3950 /* CRC has been stripped on Dune*/ 3951 skb_put(skb, pktlen); 3952 } else { 3953 skb_put(skb, pktlen - 4); /* Strip CRC */ 3954 } 3955 3956 if (device_is_sand(sinfo)) { 3957 skb_pull(skb, packet_info.system_header_size); 3958 } else if (sinfo->cmic_type == 'x') { 3959 skb_pull(skb, sinfo->pkt_hdr_size); 3960 } 3961 3962 /* Optional SKB updates */ 3963 KNET_SKB_CB(skb)->dcb_type = sinfo->dcb_type & 0xFFFF; 3964 /* Do Rx timestamping */ 3965 if (priv->rx_hwts) { 3966 bkn_hw_tstamp_rx_set(sinfo, priv->phys_port, skb, meta); 3967 } 3968 3969 /* Save for RCPU before stripping tag */ 3970 ethertype = PKT_U16_GET(skb->data, 16); 3971 if ((priv->flags & KCOM_NETIF_F_KEEP_RX_TAG) == 0) { 3972 uint16_t vlan_proto; 3973 3974 vlan_proto = PKT_U16_GET(skb->data, 12); 3975 if (filter->kf.flags & KCOM_FILTER_F_STRIP_TAG) { 3976 /* Strip VLAN tag */ 3977 if (vlan_proto == ETH_P_8021Q || 3978 vlan_proto == ETH_P_8021AD) { 3979 DBG_FLTR(("Strip VLAN tag\n")); 3980 ((u32*)skb->data)[3] = ((u32*)skb->data)[2]; 3981 ((u32*)skb->data)[2] = ((u32*)skb->data)[1]; 3982 ((u32*)skb->data)[1] = ((u32*)skb->data)[0]; 3983 skb_pull(skb, 4); 3984 if (device_is_sand(sinfo)) { 3985 for (idx = packet_info.system_header_size; idx >= 4; idx--) { 3986 pkt[idx] = pkt[idx - 4]; 3987 } 3988 } else if (sinfo->cmic_type == 'x') { 3989 for (idx = sinfo->pkt_hdr_size / sizeof(uint32_t); idx; idx--) { 3990 meta[idx] = meta[idx - 1]; 3991 } 3992 meta++; 3993 } 3994 } 3995 } else { 3996 /* 3997 * Mark packet as VLAN-tagged, otherwise newer 3998 * kernels will strip the tag. 3999 */ 4000 uint16_t tci = PKT_U16_GET(skb->data, 14); 4001 4002 if (priv->flags & KCOM_NETIF_F_RCPU_ENCAP) { 4003 bkn_vlan_hwaccel_put_tag(skb, ETH_P_8021Q, tci); 4004 } else { 4005 if (vlan_proto == ETH_P_8021AD) { 4006 bkn_vlan_hwaccel_put_tag 4007 (skb, ETH_P_8021AD, tci); 4008 } else { 4009 bkn_vlan_hwaccel_put_tag 4010 (skb, ETH_P_8021Q, tci); 4011 } 4012 } 4013 } 4014 } 4015 4016 priv->stats.rx_packets++; 4017 priv->stats.rx_bytes += skb->len; 4018 skb->dev = priv->dev; 4019 4020 if (knet_rx_cb != NULL) { 4021 KNET_SKB_CB(skb)->netif_user_data = priv->cb_user_data; 4022 KNET_SKB_CB(skb)->filter_user_data = filter->kf.cb_user_data; 4023 if (device_is_sand(sinfo)) { 4024 skb = knet_rx_cb(skb, sinfo->dev_no, sand_scratch_data); 4025 } 4026 else { 4027 skb = knet_rx_cb(skb, sinfo->dev_no, meta); 4028 } 4029 if (skb == NULL) { 4030 /* Consumed by call-back */ 4031 sinfo->rx[chan].pkts_d_callback++; 4032 priv->stats.rx_dropped++; 4033 desc->skb = NULL; 4034 break; 4035 } 4036 } 4037 4038 if (priv->flags & KCOM_NETIF_F_RCPU_ENCAP) { 4039 bkn_add_rcpu_encap(sinfo, skb, meta, packet_info.system_header_size); 4040 DBG_PDMP(("After add RCPU ENCAP\n")); 4041 bkn_dump_pkt(skb->data, pktlen + RCPU_RX_ENCAP_SIZE, XGS_DMA_RX_CHAN); 4042 } 4043 skb->protocol = eth_type_trans(skb, skb->dev); 4044 if (filter->kf.dest_proto) { 4045 skb->protocol = filter->kf.dest_proto; 4046 } 4047 if (priv->flags & KCOM_NETIF_F_RCPU_ENCAP) { 4048 bkn_eth_type_update(skb, ethertype); 4049 } 4050 DBG_DUNE(("skb protocol 0x%04x\n",skb->protocol)); 4051 4052 if (filter->kf.mirror_type == KCOM_DEST_T_NETIF) { 4053 bkn_priv_t *mpriv; 4054 struct sk_buff *mskb; 4055 mpriv = bkn_netif_lookup(sinfo, filter->kf.mirror_id); 4056 if (mpriv && netif_carrier_ok(mpriv->dev)) { 4057 mskb = skb_clone(skb, GFP_ATOMIC); 4058 if (mskb == NULL) { 4059 sinfo->rx[chan].pkts_d_no_skb++; 4060 } else { 4061 sinfo->rx[chan].pkts_m_netif++; 4062 mpriv->stats.rx_packets++; 4063 mpriv->stats.rx_bytes += mskb->len; 4064 skb->dev = mpriv->dev; 4065 if (filter->kf.mirror_proto) { 4066 skb->protocol = filter->kf.mirror_proto; 4067 } 4068 /* Unlock while calling up network stack */ 4069 spin_unlock(&sinfo->lock); 4070 if (use_napi) { 4071 netif_receive_skb(mskb); 4072 } else { 4073 netif_rx(mskb); 4074 } 4075 spin_lock(&sinfo->lock); 4076 } 4077 } 4078 } 4079 4080 /* Unlock while calling up network stack */ 4081 spin_unlock(&sinfo->lock); 4082 if (use_napi) { 4083 netif_receive_skb(skb); 4084 } else { 4085 netif_rx(skb); 4086 } 4087 spin_lock(&sinfo->lock); 4088 4089 /* Ensure that we reallocate SKB for this DCB */ 4090 desc->skb = NULL; 4091 } else { 4092 DBG_FLTR(("Unknown netif %d\n", 4093 filter->kf.dest_id)); 4094 sinfo->rx[chan].pkts_d_unkn_netif++; 4095 } 4096 break; 4097 default: 4098 /* Drop packet */ 4099 DBG_FLTR(("Unknown dest type %d\n", 4100 filter->kf.dest_type)); 4101 sinfo->rx[chan].pkts_d_unkn_dest++; 4102 break; 4103 } 4104 } else { 4105 DBG_PKT(("Rx packet dropped.\n")); 4106 sinfo->rx[chan].pkts_d_no_match++; 4107 priv->stats.rx_dropped++; 4108 } 4109 dcb[sinfo->dcb_wsize-1] &= ~(1 << 31); 4110 if (++sinfo->rx[chan].dirty >= MAX_RX_DCBS) { 4111 sinfo->rx[chan].dirty = 0; 4112 } 4113 sinfo->rx[chan].free--; 4114 dcbs_done++; 4115 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 4116 /* Right now refill for Continuous DMA mode */ 4117 bkn_rx_refill(sinfo, chan); 4118 } 4119 } 4120 4121 return dcbs_done; 4122 } 4123 4124 static int 4125 bkn_do_rx(bkn_switch_info_t *sinfo, int chan, int budget) 4126 { 4127 if (sinfo->rx[chan].use_rx_skb == 0) { 4128 /* Rx buffers are provided by BCM Rx API */ 4129 return bkn_do_api_rx(sinfo, chan, budget); 4130 } else { 4131 /* Rx buffers are provided by Linux kernel */ 4132 return bkn_do_skb_rx(sinfo, chan, budget); 4133 } 4134 } 4135 4136 static void 4137 bkn_rx_desc_done(bkn_switch_info_t *sinfo, int chan) 4138 { 4139 bkn_evt_resource_t *evt; 4140 evt = &_bkn_evt[sinfo->evt_idx]; 4141 DBG_IRQ(("Rx%d desc done\n", chan)); 4142 4143 if (sinfo->rx[chan].use_rx_skb == 0) { 4144 sinfo->dma_events |= KCOM_DMA_INFO_F_RX_DONE; 4145 evt->evt_wq_put++; 4146 wake_up_interruptible(&evt->evt_wq); 4147 } 4148 } 4149 4150 static void 4151 bkn_rx_chain_done(bkn_switch_info_t *sinfo, int chan) 4152 { 4153 DBG_IRQ(("Rx%d chain done\n", chan)); 4154 4155 if (sinfo->rx[chan].chain_complete == 0) { 4156 /* 4157 * In certain environments the DCB memory is updated after 4158 * the corresponding interrupt has been received. 4159 * The following code will ensure that this situation is 4160 * handled properly. 4161 */ 4162 int maxloop = 0; 4163 while (sinfo->rx[chan].chain_complete == 0) { 4164 sinfo->rx[chan].sync_retry++; 4165 if (maxloop == 0) { 4166 sinfo->rx[chan].sync_err++; 4167 } 4168 if (maxloop > sinfo->rx[chan].sync_maxloop) { 4169 sinfo->rx[chan].sync_maxloop = maxloop; 4170 } 4171 if (bkn_do_rx(sinfo, chan, MAX_RX_DCBS) > 0) { 4172 bkn_rx_desc_done(sinfo, chan); 4173 } 4174 if (++maxloop > rx_sync_retry) { 4175 gprintk("Fatal error: Incomplete chain\n"); 4176 sinfo->rx[chan].chain_complete = 1; 4177 break; 4178 } 4179 } 4180 } 4181 4182 sinfo->rx[chan].running = 0; 4183 4184 if (sinfo->rx[chan].use_rx_skb == 0) { 4185 bkn_api_rx_chain_done(sinfo, chan); 4186 } else { 4187 bkn_rx_refill(sinfo, chan); 4188 4189 if (bkn_rx_restart(sinfo, chan) != 0) { 4190 /* Presumably out of resources */ 4191 sinfo->timer.expires = jiffies + 1; 4192 if (!sinfo->timer_queued) { 4193 sinfo->timer_queued = 1; 4194 add_timer(&sinfo->timer); 4195 } 4196 } 4197 } 4198 } 4199 4200 static void 4201 bkn_suspend_tx(bkn_switch_info_t *sinfo) 4202 { 4203 struct list_head *list; 4204 bkn_priv_t *priv = netdev_priv(sinfo->dev); 4205 4206 /* Stop main device */ 4207 netif_stop_queue(priv->dev); 4208 sinfo->tx.suspends++; 4209 /* Stop associated virtual devices */ 4210 list_for_each(list, &sinfo->ndev_list) { 4211 priv = (bkn_priv_t *)list; 4212 netif_stop_queue(priv->dev); 4213 } 4214 } 4215 4216 static void 4217 bkn_resume_tx(bkn_switch_info_t *sinfo) 4218 { 4219 struct list_head *list; 4220 bkn_priv_t *priv = netdev_priv(sinfo->dev); 4221 4222 /* Check main device */ 4223 if (netif_queue_stopped(priv->dev) && sinfo->tx.free > 1) { 4224 netif_wake_queue(priv->dev); 4225 } 4226 /* Check associated virtual devices */ 4227 list_for_each(list, &sinfo->ndev_list) { 4228 priv = (bkn_priv_t *)list; 4229 if (netif_queue_stopped(priv->dev) && sinfo->tx.free > 1) { 4230 netif_wake_queue(priv->dev); 4231 } 4232 } 4233 } 4234 4235 static void 4236 bkn_skb_tstamp_copy(struct sk_buff *new_skb, struct sk_buff *skb) 4237 { 4238 bkn_skb_tx_flags(new_skb) = bkn_skb_tx_flags(skb); 4239 new_skb->sk = skb->sk; 4240 4241 return; 4242 } 4243 4244 static int 4245 bkn_hw_tstamp_tx_set(bkn_switch_info_t *sinfo, struct sk_buff *skb) 4246 { 4247 int hwts; 4248 int port; 4249 uint64_t ts = 0; 4250 uint32_t hdrlen = sinfo->cmic_type == 'x' ? PKT_TX_HDR_SIZE : 0; 4251 struct skb_shared_hwtstamps shhwtstamps; 4252 4253 if (!knet_hw_tstamp_tx_time_get_cb) { 4254 return -1; 4255 } 4256 4257 port = KNET_SKB_CB(skb)->port; 4258 hwts = KNET_SKB_CB(skb)->hwts; 4259 ts = KNET_SKB_CB(skb)->ts; 4260 4261 4262 if (hwts == HWTSTAMP_TX_ONESTEP_SYNC) { 4263 if (ts == 0) { 4264 return 1; 4265 } 4266 } else if (hwts == HWTSTAMP_TX_ON) { 4267 if (knet_hw_tstamp_tx_time_get_cb(sinfo->dev_no, port, skb->data + hdrlen, &ts) < 0) { 4268 return -1; 4269 } 4270 } 4271 4272 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 4273 shhwtstamps.hwtstamp = ns_to_ktime(ts); 4274 skb_tstamp_tx(skb, &shhwtstamps); 4275 4276 return 0; 4277 } 4278 4279 static void 4280 bkn_hw_tstamp_tx_work(struct work_struct *work) 4281 { 4282 bkn_switch_info_t *sinfo = container_of(work, bkn_switch_info_t, tx_ptp_work); 4283 struct sk_buff *skb; 4284 4285 while (skb_queue_len(&sinfo->tx_ptp_queue)) { 4286 skb = skb_dequeue(&sinfo->tx_ptp_queue); 4287 if (bkn_hw_tstamp_tx_set(sinfo, skb) < 0) { 4288 gprintk("Timestamp has not been taken for the current skb.\n"); 4289 } 4290 dev_kfree_skb_any(skb); 4291 } 4292 } 4293 4294 static int 4295 bkn_do_tx(bkn_switch_info_t *sinfo) 4296 { 4297 bkn_desc_info_t *desc; 4298 int dcbs_done = 0; 4299 4300 if (!CDMA_CH(sinfo, XGS_DMA_TX_CHAN) && sinfo->tx.api_active) { 4301 return dcbs_done; 4302 } 4303 4304 while (dcbs_done < MAX_TX_DCBS) { 4305 char str[32]; 4306 if (sinfo->tx.free == MAX_TX_DCBS) { 4307 break; 4308 } 4309 sprintf(str, "Tx DCB (%d)", sinfo->tx.dirty); 4310 desc = &sinfo->tx.desc[sinfo->tx.dirty]; 4311 bkn_dump_dcb(str, desc->dcb_mem, sinfo->dcb_wsize, XGS_DMA_TX_CHAN); 4312 if ((desc->dcb_mem[sinfo->dcb_wsize-1] & (1 << 31)) == 0) { 4313 break; 4314 } 4315 if (desc->skb) { 4316 DBG_DCB_TX(("Tx SKB DMA done (%d).\n", sinfo->tx.dirty)); 4317 BKN_DMA_UNMAP_SINGLE(sinfo->dma_dev, 4318 desc->skb_dma, desc->dma_size, 4319 BKN_DMA_TODEV); 4320 4321 if ((KNET_SKB_CB(desc->skb)->hwts == HWTSTAMP_TX_ONESTEP_SYNC) && 4322 (bkn_skb_tx_flags(desc->skb) & SKBTX_IN_PROGRESS)) { 4323 4324 if (bkn_hw_tstamp_tx_set(sinfo, desc->skb) < 0) { 4325 gprintk("Timestamp has not been taken for the current skb.\n"); 4326 } 4327 bkn_skb_tx_flags(desc->skb) &= ~SKBTX_IN_PROGRESS; 4328 } 4329 4330 if (bkn_skb_tx_flags(desc->skb) & SKBTX_IN_PROGRESS) { 4331 skb_queue_tail(&sinfo->tx_ptp_queue, desc->skb); 4332 schedule_work(&sinfo->tx_ptp_work); 4333 } else { 4334 dev_kfree_skb_any(desc->skb); 4335 } 4336 desc->skb = NULL; 4337 desc->skb_dma = 0; 4338 } 4339 desc->dcb_mem[sinfo->dcb_wsize-1] &= ~(1 << 31); 4340 if (++sinfo->tx.dirty >= MAX_TX_DCBS) { 4341 sinfo->tx.dirty = 0; 4342 } 4343 if (++sinfo->tx.free > MAX_TX_DCBS) { 4344 gprintk("Too many free Tx DCBs(%d).\n", sinfo->tx.free); 4345 } 4346 dcbs_done++; 4347 } 4348 4349 return dcbs_done; 4350 } 4351 4352 static void 4353 bkn_tx_cdma_chain_switch(bkn_switch_info_t *sinfo) 4354 { 4355 bkn_dcb_chain_t *dcb_chain = sinfo->tx.api_dcb_chain; 4356 uint32_t *dcb_mem; 4357 uint64_t dcb_dma; 4358 int woffset; 4359 4360 /* Switch between SKB Tx and API Tx for Continuous DMA mode */ 4361 if (!sinfo->tx.api_active) { 4362 /* 4363 * Set the current SKB DCB as reload DCB and the last DCB of 4364 * the pending API chain as the new halt location. 4365 */ 4366 sinfo->tx.api_active = 1; 4367 dcb_mem = sinfo->tx.desc[sinfo->tx.cur].dcb_mem; 4368 memset(dcb_mem, 0, sinfo->dcb_wsize * sizeof(uint32_t)); 4369 dcb_mem[0] = dcb_chain->dcb_dma; 4370 if (sinfo->cmic_type == 'x') { 4371 dcb_mem[1] = DMA_TO_BUS_HI(dcb_chain->dcb_dma >> 32); 4372 dcb_mem[2] |= 1 << 24 | 1 << 18 | 1 << 16; 4373 } else { 4374 dcb_mem[1] |= 1 << 24 | 1 << 18 | 1 << 16; 4375 } 4376 if (++sinfo->tx.cur >= MAX_TX_DCBS) { 4377 sinfo->tx.cur = 0; 4378 } 4379 sinfo->tx.free--; 4380 woffset = (dcb_chain->dcb_cnt - 1) * sinfo->dcb_wsize; 4381 dcb_dma = dcb_chain->dcb_dma + woffset * sizeof(uint32_t); 4382 /* DMA run to the new halt location */ 4383 bkn_cdma_goto(sinfo, XGS_DMA_TX_CHAN, dcb_dma); 4384 } else { 4385 /* Only need to set the current SKB DCB as the new halt location */ 4386 sinfo->tx.api_active = 0; 4387 woffset = (dcb_chain->dcb_cnt - 1) * sinfo->dcb_wsize; 4388 dcb_mem = &dcb_chain->dcb_mem[woffset]; 4389 dcb_mem[0] = sinfo->tx.desc[sinfo->tx.dirty].dcb_dma; 4390 if (sinfo->cmic_type == 'x') { 4391 dcb_mem[1] = DMA_TO_BUS_HI(sinfo->tx.desc[sinfo->tx.dirty].dcb_dma >> 32); 4392 } 4393 dcb_dma = sinfo->tx.desc[sinfo->tx.cur].dcb_dma; 4394 /* DMA run to the new halt location */ 4395 bkn_cdma_goto(sinfo, XGS_DMA_TX_CHAN, dcb_dma); 4396 } 4397 } 4398 4399 static void 4400 bkn_skb_tx(bkn_switch_info_t *sinfo) 4401 { 4402 if (sinfo->tx.api_active) { 4403 /* Switch from API Tx to SKB Tx */ 4404 bkn_tx_cdma_chain_switch(sinfo); 4405 } 4406 } 4407 4408 static void 4409 bkn_api_tx(bkn_switch_info_t *sinfo) 4410 { 4411 bkn_dcb_chain_t *dcb_chain; 4412 uint64_t pkt_dma; 4413 unsigned char *pktdata; 4414 int pktlen; 4415 int i; 4416 4417 /* Assume that driver lock is held */ 4418 if (list_empty(&sinfo->tx.api_dcb_list)) { 4419 sinfo->tx.api_active = 0; 4420 } else { 4421 sinfo->tx.pkts++; 4422 dcb_chain = list_entry(sinfo->tx.api_dcb_list.next, 4423 bkn_dcb_chain_t, list); 4424 DBG_DCB_TX(("Start API Tx DMA, first DCB @ 0x%08x (%d DCBs).\n", 4425 (uint32_t)dcb_chain->dcb_dma, dcb_chain->dcb_cnt)); 4426 for (i = 0; i < dcb_chain->dcb_cnt && debug & DBG_LVL_PDMP; i++) { 4427 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN) && i == dcb_chain->dcb_cnt - 1) { 4428 break; 4429 } 4430 if (sinfo->cmic_type == 'x') { 4431 pkt_dma = BUS_TO_DMA_HI(dcb_chain->dcb_mem[sinfo->dcb_wsize * i + 1]); 4432 pkt_dma = pkt_dma << 32 | dcb_chain->dcb_mem[sinfo->dcb_wsize * i]; 4433 } else { 4434 pkt_dma = dcb_chain->dcb_mem[sinfo->dcb_wsize * i]; 4435 } 4436 pktdata = kernel_bde->p2l(sinfo->dev_no, pkt_dma); 4437 if (sinfo->cmic_type == 'x') { 4438 pktlen = dcb_chain->dcb_mem[sinfo->dcb_wsize * i + 2] & 0xffff; 4439 } else { 4440 pktlen = dcb_chain->dcb_mem[sinfo->dcb_wsize * i + 1] & 0xffff; 4441 } 4442 bkn_dump_pkt(pktdata, pktlen, XGS_DMA_TX_CHAN); 4443 } 4444 4445 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN)) { 4446 sinfo->tx.api_dcb_chain = dcb_chain; 4447 if (!sinfo->tx.api_active) { 4448 /* Switch from SKB Tx to API Tx */ 4449 bkn_tx_cdma_chain_switch(sinfo); 4450 } 4451 list_del(&dcb_chain->list); 4452 } else { 4453 sinfo->tx.api_active = 1; 4454 dev_dma_chan_clear(sinfo, XGS_DMA_TX_CHAN); 4455 dev_irq_mask_enable(sinfo, XGS_DMA_TX_CHAN, 1); 4456 dev_dma_chan_start(sinfo, XGS_DMA_TX_CHAN, dcb_chain->dcb_dma); 4457 list_del(&dcb_chain->list); 4458 kfree(dcb_chain); 4459 } 4460 } 4461 } 4462 4463 static void 4464 bkn_tx_cdma_chain_done(bkn_switch_info_t *sinfo, int done) 4465 { 4466 int woffset; 4467 int dcbs_done = 0; 4468 bkn_evt_resource_t *evt; 4469 4470 DBG_IRQ(("Tx CDMA chain done \n")); 4471 4472 evt = &_bkn_evt[sinfo->evt_idx]; 4473 4474 if (sinfo->tx.api_active) { 4475 /* Drain API Tx chains */ 4476 while (sinfo->tx.api_dcb_chain != sinfo->tx.api_dcb_chain_end) { 4477 woffset = sinfo->tx.api_dcb_chain->dcb_cnt * sinfo->dcb_wsize - 1; 4478 if (!(sinfo->tx.api_dcb_chain->dcb_mem[woffset] & (1 << 31))) { 4479 return; 4480 } 4481 sinfo->tx.api_dcb_chain->dcb_mem[woffset - sinfo->dcb_wsize] |= SOC_DCB_KNET_DONE; 4482 sinfo->dma_events |= KCOM_DMA_INFO_F_TX_DONE; 4483 evt->evt_wq_put++; 4484 wake_up_interruptible(&evt->evt_wq); 4485 kfree(sinfo->tx.api_dcb_chain); 4486 sinfo->tx.api_dcb_chain = NULL; 4487 bkn_api_tx(sinfo); 4488 if ((++dcbs_done + done) >= MAX_TX_DCBS) { 4489 if (sinfo->napi_poll_mode) { 4490 /* Request one extra poll to reschedule Tx */ 4491 sinfo->napi_poll_again = 1; 4492 } else { 4493 /* Request to yield for Continuous DMA mode */ 4494 sinfo->tx_yield = 1; 4495 } 4496 return; 4497 } 4498 } 4499 woffset = (sinfo->tx.api_dcb_chain->dcb_cnt - 1) * sinfo->dcb_wsize - 1; 4500 if (!(sinfo->tx.api_dcb_chain->dcb_mem[woffset] & (1 << 31))) { 4501 return; 4502 } 4503 sinfo->tx.api_dcb_chain->dcb_mem[woffset] |= SOC_DCB_KNET_DONE; 4504 /* Try and park at SKB Tx if API Tx done */ 4505 bkn_skb_tx(sinfo); 4506 sinfo->dma_events |= KCOM_DMA_INFO_F_TX_DONE; 4507 evt->evt_wq_put++; 4508 wake_up_interruptible(&evt->evt_wq); 4509 kfree(sinfo->tx.api_dcb_chain); 4510 sinfo->tx.api_dcb_chain = NULL; 4511 sinfo->tx.api_dcb_chain_end = NULL; 4512 if (!sinfo->napi_poll_mode) { 4513 /* Not need to yield for Continuous DMA mode */ 4514 sinfo->tx_yield = 0; 4515 } 4516 } else { 4517 if (sinfo->tx.free == MAX_TX_DCBS) { 4518 /* Try API Tx if SKB Tx done */ 4519 bkn_api_tx(sinfo); 4520 if (sinfo->tx.api_active) { 4521 return; 4522 } 4523 } 4524 } 4525 4526 /* Resume if netif Tx resources available and API Tx not active */ 4527 bkn_resume_tx(sinfo); 4528 } 4529 4530 static void 4531 bkn_tx_chain_done(bkn_switch_info_t *sinfo, int done) 4532 { 4533 bkn_desc_info_t *desc; 4534 int idx, pending; 4535 bkn_evt_resource_t *evt; 4536 4537 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN)) { 4538 return bkn_tx_cdma_chain_done(sinfo, done); 4539 } 4540 4541 DBG_IRQ(("Tx chain done (%d/%d)\n", sinfo->tx.cur, sinfo->tx.dirty)); 4542 4543 dev_irq_mask_disable(sinfo, XGS_DMA_TX_CHAN, 0); 4544 4545 evt = &_bkn_evt[sinfo->evt_idx]; 4546 4547 if (sinfo->tx.api_active) { 4548 sinfo->dma_events |= KCOM_DMA_INFO_F_TX_DONE; 4549 evt->evt_wq_put++; 4550 wake_up_interruptible(&evt->evt_wq); 4551 /* Check if BCM API has more to send */ 4552 bkn_api_tx(sinfo); 4553 if (sinfo->tx.api_active) { 4554 return; 4555 } 4556 } 4557 4558 if (sinfo->tx.free == MAX_TX_DCBS) { 4559 /* If netif Tx is idle then allow BCM API to send */ 4560 bkn_api_tx(sinfo); 4561 if (sinfo->tx.api_active) { 4562 return; 4563 } 4564 } else { 4565 /* If two or more DCBs are pending, chain them */ 4566 pending = MAX_TX_DCBS - sinfo->tx.free; 4567 idx = sinfo->tx.dirty; 4568 while (--pending && idx < (MAX_TX_DCBS - 1)) { 4569 if (sinfo->cmic_type == 'x') { 4570 sinfo->tx.desc[idx++].dcb_mem[2] |= 1 << 16; 4571 } else { 4572 sinfo->tx.desc[idx++].dcb_mem[1] |= 1 << 16; 4573 } 4574 DBG_DCB_TX(("Chain Tx DCB %d (%d)\n", idx, pending)); 4575 } 4576 /* Restart DMA from where we stopped */ 4577 desc = &sinfo->tx.desc[sinfo->tx.dirty]; 4578 DBG_DCB_TX(("Restart Tx DMA, DCB @ 0x%08x (%d).\n", 4579 (uint32_t)desc->dcb_dma, sinfo->tx.dirty)); 4580 dev_dma_chan_clear(sinfo, XGS_DMA_TX_CHAN); 4581 dev_irq_mask_enable(sinfo, XGS_DMA_TX_CHAN, 0); 4582 dev_dma_chan_start(sinfo, XGS_DMA_TX_CHAN, desc->dcb_dma); 4583 } 4584 4585 /* Resume if netif Tx resources available and API Tx not active */ 4586 bkn_resume_tx(sinfo); 4587 } 4588 4589 static void 4590 bkn_schedule_napi_poll(bkn_switch_info_t *sinfo) 4591 { 4592 /* Schedule NAPI poll */ 4593 DBG_NAPI(("Schedule NAPI poll on %s.\n", sinfo->dev->name)); 4594 /* Disable interrupts until poll job is complete */ 4595 sinfo->napi_poll_mode = 1; 4596 /* Unlock while calling up network stack */ 4597 spin_unlock(&sinfo->lock); 4598 if (bkn_napi_schedule_prep(sinfo->dev, &sinfo->napi)) { 4599 __bkn_napi_schedule(sinfo->dev, &sinfo->napi); 4600 DBG_NAPI(("Schedule prep OK on %s.\n", sinfo->dev->name)); 4601 } else { 4602 /* Most likely the base device is has not been opened */ 4603 gprintk("Warning: Unable to schedule NAPI - base device not up?\n"); 4604 } 4605 spin_lock(&sinfo->lock); 4606 } 4607 4608 static void 4609 bkn_napi_poll_complete(bkn_switch_info_t *sinfo) 4610 { 4611 /* Unlock while calling up network stack */ 4612 spin_unlock(&sinfo->lock); 4613 bkn_napi_complete(sinfo->dev, &sinfo->napi); 4614 spin_lock(&sinfo->lock); 4615 /* Re-enable interrupts */ 4616 sinfo->napi_poll_mode = 0; 4617 dev_irq_mask_set(sinfo, sinfo->irq_mask); 4618 } 4619 4620 static int 4621 xgs_do_dma(bkn_switch_info_t *sinfo, int budget) 4622 { 4623 int rx_dcbs_done = 0, tx_dcbs_done = 0; 4624 int chan_done, budget_chans = 0; 4625 uint32_t dma_stat; 4626 int chan; 4627 4628 DEV_READ32(sinfo, CMIC_DMA_STATr, &dma_stat); 4629 4630 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4631 if (dma_stat & DS_DESC_DONE_TST(XGS_DMA_RX_CHAN + chan)) { 4632 xgs_dma_desc_clear(sinfo, XGS_DMA_RX_CHAN + chan); 4633 sinfo->poll_channels |= 1 << chan; 4634 } 4635 } 4636 if (!sinfo->poll_channels) { 4637 sinfo->poll_channels = (uint32_t)(1 << sinfo->rx_chans) - 1; 4638 budget_chans = budget / sinfo->rx_chans; 4639 } else { 4640 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4641 if (1 << chan & sinfo->poll_channels) { 4642 budget_chans++; 4643 } 4644 } 4645 budget_chans = budget / budget_chans; 4646 } 4647 4648 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4649 if (1 << chan & sinfo->poll_channels) { 4650 chan_done = bkn_do_rx(sinfo, chan, budget_chans); 4651 rx_dcbs_done += chan_done; 4652 if (chan_done < budget_chans) { 4653 sinfo->poll_channels &= ~(1 << chan); 4654 } 4655 bkn_rx_desc_done(sinfo, chan); 4656 } 4657 4658 if (dma_stat & DS_CHAIN_DONE_TST(XGS_DMA_RX_CHAN + chan)) { 4659 xgs_dma_chain_clear(sinfo, XGS_DMA_RX_CHAN + chan); 4660 bkn_rx_chain_done(sinfo, chan); 4661 } 4662 } 4663 4664 if (dma_stat & DS_CHAIN_DONE_TST(XGS_DMA_TX_CHAN)) { 4665 xgs_dma_chain_clear(sinfo, XGS_DMA_TX_CHAN); 4666 tx_dcbs_done = bkn_do_tx(sinfo); 4667 bkn_tx_chain_done(sinfo, tx_dcbs_done); 4668 } 4669 4670 return sinfo->poll_channels ? budget : rx_dcbs_done; 4671 } 4672 4673 static int 4674 xgsm_do_dma(bkn_switch_info_t *sinfo, int budget) 4675 { 4676 int rx_dcbs_done = 0, tx_dcbs_done = 0; 4677 int chan_done, budget_chans = 0; 4678 uint32_t dma_stat, irq_stat = 0; 4679 int chan; 4680 4681 /* Get Controlled interrupt states for Continuous DMA mode */ 4682 if (sinfo->cdma_channels) { 4683 DEV_READ32(sinfo, CMICM_IRQ_STATr, &irq_stat); 4684 } 4685 4686 DEV_READ32(sinfo, CMICM_DMA_STATr, &dma_stat); 4687 4688 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4689 if (dma_stat & (0x10 << (XGS_DMA_RX_CHAN + chan)) || 4690 irq_stat & (0x08000000 << (XGS_DMA_RX_CHAN + chan))) { 4691 xgsm_dma_desc_clear(sinfo, XGS_DMA_RX_CHAN + chan); 4692 sinfo->poll_channels |= 1 << chan; 4693 } 4694 } 4695 if (!sinfo->poll_channels) { 4696 sinfo->poll_channels = (uint32_t)(1 << sinfo->rx_chans) - 1; 4697 budget_chans = budget / sinfo->rx_chans; 4698 } else { 4699 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4700 if (1 << chan & sinfo->poll_channels) { 4701 budget_chans++; 4702 } 4703 } 4704 budget_chans = budget / budget_chans; 4705 } 4706 4707 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4708 if (1 << chan & sinfo->poll_channels) { 4709 chan_done = bkn_do_rx(sinfo, chan, budget_chans); 4710 rx_dcbs_done += chan_done; 4711 if (chan_done < budget_chans) { 4712 sinfo->poll_channels &= ~(1 << chan); 4713 } 4714 bkn_rx_desc_done(sinfo, chan); 4715 } 4716 4717 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 4718 continue; 4719 } 4720 4721 if (dma_stat & (0x1 << (XGS_DMA_RX_CHAN + chan))) { 4722 xgsm_dma_chain_clear(sinfo, XGS_DMA_RX_CHAN + chan); 4723 bkn_rx_chain_done(sinfo, chan); 4724 } 4725 } 4726 4727 if (dma_stat & (0x1 << XGS_DMA_TX_CHAN) || 4728 irq_stat & (0x08000000 << XGS_DMA_TX_CHAN)) { 4729 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN)) { 4730 xgsm_dma_desc_clear(sinfo, XGS_DMA_TX_CHAN); 4731 } else { 4732 xgsm_dma_chain_clear(sinfo, XGS_DMA_TX_CHAN); 4733 } 4734 tx_dcbs_done = bkn_do_tx(sinfo); 4735 bkn_tx_chain_done(sinfo, tx_dcbs_done); 4736 } 4737 4738 return sinfo->poll_channels ? budget : rx_dcbs_done; 4739 } 4740 4741 static int 4742 xgsx_do_dma(bkn_switch_info_t *sinfo, int budget) 4743 { 4744 int rx_dcbs_done = 0, tx_dcbs_done = 0; 4745 int chan_done, budget_chans = 0; 4746 uint32_t irq_stat, tx_dma_stat, rx_dma_stat[NUM_CMICX_RX_CHAN]; 4747 int chan; 4748 4749 DEV_READ32(sinfo, CMICX_IRQ_STATr, &irq_stat); 4750 DEV_READ32(sinfo, CMICX_DMA_STATr + 0x80 * XGS_DMA_TX_CHAN, &tx_dma_stat); 4751 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4752 DEV_READ32(sinfo, 4753 CMICX_DMA_STATr + 0x80 * (XGS_DMA_RX_CHAN + chan), 4754 &rx_dma_stat[chan]); 4755 } 4756 4757 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4758 if ((irq_stat & CMICX_DS_CMC_CTRLD_INT(XGS_DMA_RX_CHAN + chan)) || 4759 (irq_stat & CMICX_DS_CMC_DESC_DONE(XGS_DMA_RX_CHAN + chan))) { 4760 xgsx_dma_desc_clear(sinfo, XGS_DMA_RX_CHAN + chan); 4761 sinfo->poll_channels |= 1 << chan; 4762 } 4763 } 4764 if (!sinfo->poll_channels) { 4765 sinfo->poll_channels = (uint32_t)(1 << sinfo->rx_chans) - 1; 4766 budget_chans = budget / sinfo->rx_chans; 4767 } else { 4768 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4769 if (1 << chan & sinfo->poll_channels) { 4770 budget_chans++; 4771 } 4772 } 4773 budget_chans = budget / budget_chans; 4774 } 4775 4776 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4777 if (1 << chan & sinfo->poll_channels) { 4778 chan_done = bkn_do_rx(sinfo, chan, budget_chans); 4779 rx_dcbs_done += chan_done; 4780 if (chan_done < budget_chans) { 4781 sinfo->poll_channels &= ~(1 << chan); 4782 } 4783 bkn_rx_desc_done(sinfo, chan); 4784 } 4785 4786 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 4787 continue; 4788 } 4789 4790 if (rx_dma_stat[chan] & CMICX_DS_CMC_DMA_CHAIN_DONE) { 4791 xgsx_dma_chain_clear(sinfo, XGS_DMA_RX_CHAN + chan); 4792 bkn_rx_chain_done(sinfo, chan); 4793 } 4794 } 4795 4796 if ((irq_stat & CMICX_DS_CMC_CTRLD_INT(XGS_DMA_TX_CHAN)) || 4797 (tx_dma_stat & CMICX_DS_CMC_DMA_CHAIN_DONE)) { 4798 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN)) { 4799 xgsx_dma_desc_clear(sinfo, XGS_DMA_TX_CHAN); 4800 } else { 4801 xgsx_dma_chain_clear(sinfo, XGS_DMA_TX_CHAN); 4802 } 4803 tx_dcbs_done = bkn_do_tx(sinfo); 4804 bkn_tx_chain_done(sinfo, tx_dcbs_done); 4805 } 4806 4807 return sinfo->poll_channels ? budget : rx_dcbs_done; 4808 } 4809 4810 static int 4811 dev_do_dma(bkn_switch_info_t *sinfo, int budget) 4812 { 4813 if (DEV_IS_CMICX(sinfo)) { 4814 return xgsx_do_dma(sinfo, budget); 4815 } else if (DEV_IS_CMICM(sinfo)) { 4816 return xgsm_do_dma(sinfo, budget); 4817 } else { 4818 return xgs_do_dma(sinfo, budget); 4819 } 4820 } 4821 4822 static void 4823 xgs_isr(bkn_switch_info_t *sinfo) 4824 { 4825 uint32_t irq_stat; 4826 int rx_dcbs_done; 4827 4828 DEV_READ32(sinfo, CMIC_IRQ_STATr, &irq_stat); 4829 if ((irq_stat & sinfo->irq_mask) == 0) { 4830 /* Not ours */ 4831 return; 4832 } 4833 sinfo->interrupts++; 4834 4835 DBG_IRQ(("Got interrupt on device %d (0x%08x)\n", 4836 sinfo->dev_no, irq_stat)); 4837 4838 if (use_napi) { 4839 bkn_schedule_napi_poll(sinfo); 4840 } else { 4841 xgs_irq_mask_set(sinfo, 0); 4842 do { 4843 rx_dcbs_done = xgs_do_dma(sinfo, MAX_RX_DCBS); 4844 } while (rx_dcbs_done); 4845 } 4846 4847 xgs_irq_mask_set(sinfo, sinfo->irq_mask); 4848 } 4849 4850 static void 4851 xgsm_isr(bkn_switch_info_t *sinfo) 4852 { 4853 uint32_t irq_stat; 4854 int rx_dcbs_done; 4855 4856 DEV_READ32(sinfo, CMICM_IRQ_STATr, &irq_stat); 4857 if ((irq_stat & sinfo->irq_mask) == 0) { 4858 /* Not ours */ 4859 return; 4860 } 4861 sinfo->interrupts++; 4862 4863 DBG_IRQ(("Got interrupt on device %d (0x%08x)\n", 4864 sinfo->dev_no, irq_stat)); 4865 4866 if (use_napi) { 4867 bkn_schedule_napi_poll(sinfo); 4868 } else { 4869 xgsm_irq_mask_set(sinfo, 0); 4870 do { 4871 rx_dcbs_done = xgsm_do_dma(sinfo, MAX_RX_DCBS); 4872 if (sinfo->cdma_channels) { 4873 if (rx_dcbs_done >= MAX_RX_DCBS || sinfo->tx_yield) { 4874 /* Continuous DMA mode requires to yield timely */ 4875 break; 4876 } 4877 } 4878 } while (rx_dcbs_done); 4879 } 4880 4881 xgsm_irq_mask_set(sinfo, sinfo->irq_mask); 4882 } 4883 4884 static void 4885 xgsx_isr(bkn_switch_info_t *sinfo) 4886 { 4887 uint32_t irq_stat; 4888 int rx_dcbs_done; 4889 4890 DEV_READ32(sinfo, CMICX_IRQ_STATr, &irq_stat); 4891 if ((irq_stat & sinfo->irq_mask) == 0) { 4892 /* Not ours */ 4893 return; 4894 } 4895 4896 /* Bypass chain_done from Abort */ 4897 if (device_is_dnx(sinfo)) { 4898 uint32_t ctrl = 0; 4899 int chan = 0; 4900 for (chan = 0; chan < NUM_DMA_CHAN; chan++) { 4901 if (irq_stat & CMICX_DS_CMC_CHAIN_DONE(chan)) { 4902 DEV_READ32(sinfo, CMICX_DMA_CTRLr + 0x80 * chan, &ctrl); 4903 if (ctrl & CMICX_DC_CMC_ABORT) { 4904 DBG_IRQ(("chain %d: chain done for Abort\n", chan)); 4905 return; 4906 } 4907 } 4908 } 4909 } 4910 4911 sinfo->interrupts++; 4912 4913 DBG_IRQ(("Got interrupt on device %d (0x%08x)\n", 4914 sinfo->dev_no, irq_stat)); 4915 4916 if (use_napi) { 4917 bkn_schedule_napi_poll(sinfo); 4918 } else { 4919 xgsx_irq_mask_set(sinfo, 0); 4920 do { 4921 rx_dcbs_done = xgsx_do_dma(sinfo, MAX_RX_DCBS); 4922 if (sinfo->cdma_channels) { 4923 if (rx_dcbs_done >= MAX_RX_DCBS || sinfo->tx_yield) { 4924 /* Continuous DMA mode requires to yield timely */ 4925 break; 4926 } 4927 } 4928 } while (rx_dcbs_done); 4929 } 4930 4931 xgsx_irq_mask_set(sinfo, sinfo->irq_mask); 4932 } 4933 4934 static void 4935 bkn_isr(void *isr_data) 4936 { 4937 bkn_switch_info_t *sinfo = isr_data; 4938 4939 /* Safe exit on SMP systems */ 4940 if (!module_initialized) { 4941 return; 4942 } 4943 4944 /* Ensure that we do not touch registers during device reset */ 4945 if (sinfo->irq_mask == 0) { 4946 /* Not ours */ 4947 return; 4948 } 4949 4950 spin_lock(&sinfo->lock); 4951 4952 if (sinfo->napi_poll_mode) { 4953 /* Not ours */ 4954 spin_unlock(&sinfo->lock); 4955 return; 4956 } 4957 4958 if (DEV_IS_CMICX(sinfo)) { 4959 xgsx_isr(sinfo); 4960 } else if (DEV_IS_CMICM(sinfo)) { 4961 xgsm_isr(sinfo); 4962 } else { 4963 xgs_isr(sinfo); 4964 } 4965 4966 spin_unlock(&sinfo->lock); 4967 } 4968 4969 #ifdef CONFIG_NET_POLL_CONTROLLER 4970 static void 4971 bkn_poll_controller(struct net_device *dev) 4972 { 4973 bkn_priv_t *priv = netdev_priv(dev); 4974 4975 disable_irq(dev->irq); 4976 bkn_isr(priv->sinfo); 4977 enable_irq(dev->irq); 4978 } 4979 #endif 4980 4981 static void 4982 bkn_resume_rx(bkn_switch_info_t *sinfo) 4983 { 4984 bkn_desc_info_t *desc; 4985 bkn_dcb_chain_t *dcb_chain; 4986 uint64_t cur_halt, last_dcb, dcb_dma; 4987 int woffset, chan, cdma_running; 4988 4989 /* Resume Rx DMA on all channels */ 4990 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4991 if (sinfo->rx[chan].use_rx_skb) { 4992 cdma_running = 0; 4993 bkn_api_rx_restart(sinfo, chan); 4994 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 4995 cur_halt = sinfo->halt_addr[XGS_DMA_RX_CHAN + chan]; 4996 last_dcb = sinfo->rx[chan].desc[MAX_RX_DCBS].dcb_dma; 4997 if (cur_halt != last_dcb) { 4998 desc = &sinfo->rx[chan].desc[sinfo->rx[chan].dirty + 1]; 4999 bkn_cdma_goto(sinfo, XGS_DMA_RX_CHAN + chan, desc->dcb_dma); 5000 cdma_running = 1; 5001 } 5002 } 5003 if (!cdma_running) { 5004 bkn_rx_restart(sinfo, chan); 5005 } 5006 } else { 5007 cdma_running = 0; 5008 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 5009 if (sinfo->rx[chan].api_active) { 5010 dcb_chain = sinfo->rx[chan].api_dcb_chain_end; 5011 woffset = (dcb_chain->dcb_cnt - 1) * sinfo->dcb_wsize; 5012 dcb_dma = dcb_chain->dcb_dma + woffset * sizeof(uint32_t); 5013 bkn_cdma_goto(sinfo, XGS_DMA_RX_CHAN + chan, dcb_dma); 5014 cdma_running = 1; 5015 } 5016 } 5017 if (!cdma_running) { 5018 bkn_api_rx_restart(sinfo, chan); 5019 } 5020 } 5021 } 5022 } 5023 5024 static int 5025 bkn_open(struct net_device *dev) 5026 { 5027 bkn_priv_t *priv = netdev_priv(dev); 5028 bkn_switch_info_t *sinfo = priv->sinfo; 5029 unsigned long flags; 5030 5031 /* Check if base device */ 5032 if (priv->id <= 0) { 5033 /* NAPI used only on base device */ 5034 if (use_napi) { 5035 bkn_napi_enable(dev, &sinfo->napi); 5036 } 5037 5038 /* Start DMA when base device is started */ 5039 if (sinfo->basedev_suspended) { 5040 spin_lock_irqsave(&sinfo->lock, flags); 5041 dev_do_dma(sinfo, MAX_RX_DCBS); 5042 sinfo->basedev_suspended = 0; 5043 bkn_api_tx(sinfo); 5044 if (!sinfo->tx.api_active) { 5045 bkn_resume_tx(sinfo); 5046 } 5047 bkn_resume_rx(sinfo); 5048 spin_unlock_irqrestore(&sinfo->lock, flags); 5049 } 5050 } 5051 5052 if (!sinfo->basedev_suspended) { 5053 netif_start_queue(dev); 5054 } 5055 5056 return 0; 5057 } 5058 5059 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29)) 5060 static int 5061 bkn_set_mac_address(struct net_device *dev, void *addr) 5062 { 5063 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,18,12)) 5064 if (!is_valid_ether_addr((const u8*)(((struct sockaddr *)addr)->sa_data))) { 5065 #else 5066 if (!is_valid_ether_addr(((struct sockaddr *)addr)->sa_data)) { 5067 #endif 5068 return -EINVAL; 5069 } 5070 memcpy(dev->dev_addr, ((struct sockaddr *)addr)->sa_data, dev->addr_len); 5071 return 0; 5072 } 5073 #endif 5074 5075 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29)) 5076 static int 5077 bkn_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 5078 { 5079 bkn_priv_t *priv = netdev_priv(dev); 5080 bkn_switch_info_t *sinfo = priv->sinfo; 5081 struct hwtstamp_config config; 5082 5083 if (cmd == SIOCSHWTSTAMP) { 5084 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) { 5085 return -EFAULT; 5086 } 5087 5088 if (!knet_hw_tstamp_enable_cb || !knet_hw_tstamp_disable_cb || 5089 priv->type != KCOM_NETIF_T_PORT) { 5090 return -ENOSYS; 5091 } 5092 5093 switch (config.tx_type) { 5094 case HWTSTAMP_TX_OFF: 5095 knet_hw_tstamp_disable_cb(sinfo->dev_no, priv->phys_port, config.tx_type); 5096 priv->tx_hwts = (config.tx_type); 5097 break; 5098 case HWTSTAMP_TX_ON: 5099 knet_hw_tstamp_enable_cb(sinfo->dev_no, priv->phys_port, config.tx_type); 5100 priv->tx_hwts = (config.tx_type); 5101 break; 5102 case HWTSTAMP_TX_ONESTEP_SYNC: 5103 knet_hw_tstamp_enable_cb(sinfo->dev_no, priv->phys_port, config.tx_type); 5104 priv->tx_hwts = (config.tx_type); 5105 break; 5106 default: 5107 return -ERANGE; 5108 } 5109 5110 switch (config.rx_filter) { 5111 case HWTSTAMP_FILTER_NONE: 5112 if (priv->rx_hwts) { 5113 knet_hw_tstamp_disable_cb(sinfo->dev_no, priv->phys_port, config.tx_type); 5114 priv->rx_hwts = 0; 5115 } 5116 break; 5117 default: 5118 if (!priv->rx_hwts) { 5119 knet_hw_tstamp_enable_cb(sinfo->dev_no, priv->phys_port, config.tx_type); 5120 priv->rx_hwts = 1; 5121 } 5122 config.rx_filter = HWTSTAMP_FILTER_ALL; 5123 break; 5124 } 5125 5126 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? -EFAULT : 0; 5127 } 5128 5129 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0)) 5130 if (cmd == SIOCGHWTSTAMP) { 5131 config.flags = 0; 5132 config.tx_type = priv->tx_hwts; 5133 config.rx_filter = priv->rx_hwts ? HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 5134 5135 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? -EFAULT : 0; 5136 } 5137 #endif 5138 5139 return -EINVAL; 5140 } 5141 #endif 5142 5143 static int 5144 bkn_change_mtu(struct net_device *dev, int new_mtu) 5145 { 5146 int max_size = new_mtu + ETH_HLEN + VLAN_HLEN + 4; 5147 5148 if (new_mtu < 68 || max_size > rx_buffer_size) { 5149 return -EINVAL; 5150 } 5151 dev->mtu = new_mtu; 5152 return 0; 5153 } 5154 5155 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) 5156 static int 5157 bkn_poll(struct net_device *dev, int *budget) 5158 { 5159 bkn_priv_t *priv = netdev_priv(dev); 5160 bkn_switch_info_t *sinfo = priv->sinfo; 5161 int cur_budget = *budget; 5162 int poll_again = 0; 5163 int rx_dcbs_done; 5164 unsigned long flags; 5165 5166 spin_lock_irqsave(&sinfo->lock, flags); 5167 5168 DBG_NAPI(("NAPI poll on %s.\n", dev->name)); 5169 5170 sinfo->napi_poll_again = 0; 5171 5172 if (cur_budget > dev->quota) { 5173 cur_budget = dev->quota; 5174 } 5175 5176 rx_dcbs_done = dev_do_dma(sinfo, cur_budget); 5177 5178 *budget -= rx_dcbs_done; 5179 cur_budget -= rx_dcbs_done; 5180 dev->quota -= rx_dcbs_done; 5181 5182 if (sinfo->napi_poll_again || cur_budget <= 0) { 5183 poll_again = 1; 5184 sinfo->napi_not_done++; 5185 } else { 5186 bkn_napi_poll_complete(sinfo); 5187 } 5188 5189 spin_unlock_irqrestore(&sinfo->lock, flags); 5190 5191 return poll_again; 5192 } 5193 #else 5194 static int 5195 bkn_poll(struct napi_struct *napi, int budget) 5196 { 5197 bkn_switch_info_t *sinfo = container_of(napi, bkn_switch_info_t, napi); 5198 int rx_dcbs_done; 5199 unsigned long flags; 5200 5201 spin_lock_irqsave(&sinfo->lock, flags); 5202 5203 DBG_NAPI(("NAPI poll on %s.\n", sinfo->dev->name)); 5204 5205 sinfo->napi_poll_again = 0; 5206 5207 rx_dcbs_done = dev_do_dma(sinfo, budget); 5208 5209 if (sinfo->napi_poll_again || rx_dcbs_done >= budget) { 5210 /* Force poll again */ 5211 rx_dcbs_done = budget; 5212 sinfo->napi_not_done++; 5213 } else { 5214 bkn_napi_poll_complete(sinfo); 5215 } 5216 5217 spin_unlock_irqrestore(&sinfo->lock, flags); 5218 5219 return rx_dcbs_done; 5220 } 5221 #endif 5222 5223 static int 5224 bkn_stop(struct net_device *dev) 5225 { 5226 bkn_priv_t *priv = netdev_priv(dev); 5227 bkn_switch_info_t *sinfo = priv->sinfo; 5228 unsigned long flags; 5229 5230 netif_stop_queue(dev); 5231 5232 /* Check if base device */ 5233 if (priv->id <= 0) { 5234 /* NAPI used only on base device */ 5235 if (use_napi) { 5236 bkn_napi_disable(dev, &sinfo->napi); 5237 } 5238 /* Suspend all devices if base device is stopped */ 5239 if (basedev_suspend) { 5240 spin_lock_irqsave(&sinfo->lock, flags); 5241 bkn_suspend_tx(sinfo); 5242 sinfo->basedev_suspended = 1; 5243 spin_unlock_irqrestore(&sinfo->lock, flags); 5244 } 5245 } 5246 5247 return 0; 5248 } 5249 5250 /* 5251 * Network Device Statistics. 5252 * Cleared at init time. 5253 */ 5254 static struct net_device_stats * 5255 bkn_get_stats(struct net_device *dev) 5256 { 5257 bkn_priv_t *priv = netdev_priv(dev); 5258 5259 return &priv->stats; 5260 } 5261 5262 /* Fake multicast ability */ 5263 static void 5264 bkn_set_multicast_list(struct net_device *dev) 5265 { 5266 } 5267 5268 static int 5269 bkn_hw_tstamp_tx_config(bkn_switch_info_t *sinfo, 5270 int hwts, int hdrlen, 5271 struct sk_buff *skb, u32 *meta) 5272 { 5273 uint32_t *md = NULL; 5274 5275 if (!knet_hw_tstamp_tx_meta_get_cb) { 5276 return -1; 5277 } 5278 5279 KNET_SKB_CB(skb)->dcb_type = sinfo->dcb_type & 0xFFFF; 5280 knet_hw_tstamp_tx_meta_get_cb(sinfo->dev_no, hwts, hdrlen, skb, 5281 &(KNET_SKB_CB(skb)->ts), (meta ? &md : NULL)); 5282 5283 if (!meta) { 5284 return 0; 5285 } 5286 5287 if (!md) { 5288 return -1; 5289 } 5290 5291 switch (sinfo->dcb_type) { 5292 case 26: 5293 case 32: 5294 case 33: 5295 meta[2] |= md[0]; 5296 meta[3] |= md[1]; 5297 meta[4] |= md[2]; 5298 meta[5] |= md[3]; 5299 break; 5300 case 36: 5301 case 38: 5302 meta[0] |= htonl(md[0]); 5303 meta[1] |= htonl(md[1]); 5304 meta[2] |= htonl(md[2]); 5305 meta[3] |= htonl(md[3]); 5306 break; 5307 default: 5308 return -1; 5309 } 5310 5311 return 0; 5312 } 5313 5314 static int 5315 bkn_tx(struct sk_buff *skb, struct net_device *dev) 5316 { 5317 bkn_priv_t *priv = netdev_priv(dev); 5318 bkn_switch_info_t *sinfo = priv->sinfo; 5319 struct sk_buff *new_skb = NULL; 5320 unsigned char *pktdata; 5321 int pktlen, hdrlen, taglen, rcpulen, metalen; 5322 int sop, idx; 5323 uint16_t tpid; 5324 uint32_t *metadata; 5325 unsigned long flags; 5326 uint32_t cpu_channel = 0; 5327 int headroom, tailroom; 5328 5329 DBG_VERB(("Netif Tx: Len=%d priv->id=%d\n", skb->len, priv->id)); 5330 5331 if (priv->id <= 0) { 5332 /* Do not transmit on base device */ 5333 priv->stats.tx_dropped++; 5334 dev_kfree_skb_any(skb); 5335 return 0; 5336 } 5337 5338 if (device_is_dnx(sinfo) && (skb->len == 0)) { 5339 priv->stats.tx_dropped++; 5340 dev_kfree_skb_any(skb); 5341 return 0; 5342 } 5343 5344 if (!netif_carrier_ok(dev)) { 5345 DBG_WARN(("Tx drop: Netif link is down.\n")); 5346 priv->stats.tx_dropped++; 5347 sinfo->tx.pkts_d_no_link++; 5348 dev_kfree_skb_any(skb); 5349 return 0; 5350 } 5351 5352 spin_lock_irqsave(&sinfo->lock, flags); 5353 5354 if (sinfo->tx.free > 1) { 5355 bkn_desc_info_t *desc = &sinfo->tx.desc[sinfo->tx.cur]; 5356 uint32_t *dcb, *meta; 5357 5358 pktdata = skb->data; 5359 pktlen = skb->len; 5360 5361 if (device_is_sand(sinfo)) { 5362 hdrlen = priv->system_headers_size; 5363 } 5364 else { 5365 hdrlen = (sinfo->cmic_type == 'x' ) ? PKT_TX_HDR_SIZE : 0; 5366 } 5367 rcpulen = 0; 5368 sop = 0; 5369 5370 if (priv->flags & KCOM_NETIF_F_RCPU_ENCAP) { 5371 rcpulen = RCPU_HDR_SIZE; 5372 if (skb->len < (rcpulen + 14)) { 5373 DBG_WARN(("Tx drop: Invalid RCPU encapsulation\n")); 5374 priv->stats.tx_dropped++; 5375 sinfo->tx.pkts_d_rcpu_encap++; 5376 dev_kfree_skb_any(skb); 5377 spin_unlock_irqrestore(&sinfo->lock, flags); 5378 return 0; 5379 } 5380 if (check_rcpu_signature && 5381 PKT_U16_GET(skb->data, 18) != sinfo->rcpu_sig) { 5382 DBG_WARN(("Tx drop: Invalid RCPU signature\n")); 5383 priv->stats.tx_dropped++; 5384 sinfo->tx.pkts_d_rcpu_sig++; 5385 dev_kfree_skb_any(skb); 5386 spin_unlock_irqrestore(&sinfo->lock, flags); 5387 return 0; 5388 } 5389 5390 if (device_is_sand(sinfo)) { 5391 /* Dune devices don't use meta data */ 5392 sop = 0; 5393 /* Get CPU channel from rcpu_hdr_t.reserved */ 5394 cpu_channel = (skb->data[28] << 24) | (skb->data[29] << 16) | (skb->data[30] << 8) | (skb->data[31]); 5395 /* System headers are supposed to be set by users in RCPU mode. */ 5396 hdrlen = 0; 5397 } else if (skb->data[21] & RCPU_F_MODHDR) { 5398 sop = skb->data[RCPU_HDR_SIZE]; 5399 switch (sop) { 5400 case 0xff: 5401 case 0x81: 5402 case 0xfb: 5403 case 0xfc: 5404 break; 5405 default: 5406 DBG_WARN(("Tx drop: Invalid RCPU meta data\n")); 5407 priv->stats.tx_dropped++; 5408 sinfo->tx.pkts_d_rcpu_meta++; 5409 dev_kfree_skb_any(skb); 5410 spin_unlock_irqrestore(&sinfo->lock, flags); 5411 return 0; 5412 } 5413 if (sinfo->cmic_type != 'x') { 5414 rcpulen += RCPU_TX_META_SIZE; 5415 } 5416 } 5417 /* Skip over RCPU encapsulation */ 5418 pktdata = &skb->data[rcpulen]; 5419 pktlen -= rcpulen; 5420 5421 /* CPU packets require tag */ 5422 if (sop == 0) { 5423 if (device_is_sand(sinfo)) { 5424 /* 5425 * There should be Module Header + PTCH_2 + [ITMH] on JR2, 5426 * PTCH_2 +[ITMH] on JR1 5427 */ 5428 } else { 5429 hdrlen = 0; 5430 tpid = PKT_U16_GET(pktdata, 12); 5431 if (tpid != 0x8100) { 5432 if (skb_header_cloned(skb)) { 5433 /* Current SKB cannot be modified */ 5434 DBG_SKB(("Realloc Tx SKB\n")); 5435 /* 5436 * New SKB needs extra TAG_SZ for VLAN tag 5437 * and extra FCS_SZ for Ethernet FCS. 5438 */ 5439 headroom = TAG_SZ; 5440 tailroom = FCS_SZ; 5441 new_skb = skb_copy_expand(skb, 5442 headroom + skb_headroom(skb), 5443 tailroom + skb_tailroom(skb), 5444 GFP_ATOMIC); 5445 if (new_skb == NULL) { 5446 DBG_WARN(("Tx drop: No SKB memory\n")); 5447 priv->stats.tx_dropped++; 5448 sinfo->tx.pkts_d_no_skb++; 5449 dev_kfree_skb_any(skb); 5450 spin_unlock_irqrestore(&sinfo->lock, flags); 5451 return 0; 5452 } 5453 /* Remove rcpulen from buffer. */ 5454 skb_pull(new_skb, rcpulen); 5455 /* Extended by TAG_SZ at the start of buffer. */ 5456 skb_push(new_skb, TAG_SZ); 5457 /* Restore the data before the tag. */ 5458 memcpy(new_skb->data, pktdata, 12); 5459 bkn_skb_tstamp_copy(new_skb, skb); 5460 dev_kfree_skb_any(skb); 5461 skb = new_skb; 5462 pktdata = skb->data; 5463 rcpulen = 0; 5464 } else { 5465 /* Add tag to RCPU header space */ 5466 DBG_SKB(("Expand into unused RCPU header\n")); 5467 rcpulen -= TAG_SZ; 5468 pktdata = &skb->data[rcpulen]; 5469 for (idx = 0; idx < 12; idx++) { 5470 pktdata[idx] = pktdata[idx + TAG_SZ]; 5471 } 5472 } 5473 pktdata[12] = 0x81; 5474 pktdata[13] = 0x00; 5475 pktdata[14] = (priv->vlan >> 8) & 0xf; 5476 pktdata[15] = priv->vlan & 0xff; 5477 pktlen += TAG_SZ; 5478 } 5479 } 5480 } 5481 } else { 5482 if (((sinfo->cmic_type == 'x') && (priv->port >= 0)) 5483 || device_is_sand(sinfo)) { 5484 if (skb_header_cloned(skb) || skb_headroom(skb) < hdrlen + 4) { 5485 /* Current SKB cannot be modified */ 5486 DBG_SKB(("Realloc Tx SKB\n")); 5487 if (device_is_sand(sinfo)) { 5488 headroom = hdrlen; 5489 } else { 5490 headroom = hdrlen + 4; 5491 } 5492 tailroom = FCS_SZ; 5493 new_skb = skb_copy_expand(skb, 5494 headroom + skb_headroom(skb), 5495 tailroom + skb_tailroom(skb), 5496 GFP_ATOMIC); 5497 if (new_skb == NULL) { 5498 DBG_WARN(("Tx drop: No SKB memory\n")); 5499 priv->stats.tx_dropped++; 5500 sinfo->tx.pkts_d_no_skb++; 5501 dev_kfree_skb_any(skb); 5502 spin_unlock_irqrestore(&sinfo->lock, flags); 5503 return 0; 5504 } 5505 skb_push(new_skb, hdrlen); 5506 bkn_skb_tstamp_copy(new_skb, skb); 5507 dev_kfree_skb_any(skb); 5508 skb = new_skb; 5509 } else { 5510 DBG_SKB(("Expand Tx SKB\n")); 5511 skb_push(skb, hdrlen); 5512 } 5513 memset(skb->data, 0, hdrlen); 5514 pktdata = skb->data; 5515 pktlen += hdrlen; 5516 } else { 5517 hdrlen = 0; 5518 } 5519 5520 if (priv->port < 0 || (priv->flags & KCOM_NETIF_F_ADD_TAG)) { 5521 DBG_DUNE(("ADD VLAN TAG\n")); 5522 /* Need to add VLAN tag if packet is untagged */ 5523 tpid = PKT_U16_GET(skb->data, hdrlen + 12); 5524 if (tpid != 0x8100) { 5525 if (skb_header_cloned(skb) || skb_headroom(skb) < 4) { 5526 /* Current SKB cannot be modified */ 5527 DBG_SKB(("Realloc Tx SKB\n")); 5528 headroom = TAG_SZ; 5529 tailroom = FCS_SZ; 5530 new_skb = skb_copy_expand(skb, 5531 headroom + skb_headroom(skb), 5532 tailroom + skb_tailroom(skb), 5533 GFP_ATOMIC); 5534 if (new_skb == NULL) { 5535 DBG_WARN(("Tx drop: No SKB memory\n")); 5536 priv->stats.tx_dropped++; 5537 sinfo->tx.pkts_d_no_skb++; 5538 dev_kfree_skb_any(skb); 5539 spin_unlock_irqrestore(&sinfo->lock, flags); 5540 return 0; 5541 } 5542 skb_push(new_skb, TAG_SZ); 5543 memcpy(new_skb->data, pktdata, hdrlen + 12); 5544 bkn_skb_tstamp_copy(new_skb, skb); 5545 dev_kfree_skb_any(skb); 5546 skb = new_skb; 5547 } else { 5548 /* Add tag to existing buffer */ 5549 DBG_SKB(("Expand Tx SKB\n")); 5550 skb_push(skb, TAG_SZ); 5551 for (idx = 0; idx < hdrlen + 12; idx++) { 5552 skb->data[idx] = skb->data[idx + TAG_SZ]; 5553 } 5554 } 5555 pktdata = skb->data; 5556 pktdata[hdrlen + 12] = 0x81; 5557 pktdata[hdrlen + 13] = 0x00; 5558 pktdata[hdrlen + 14] = (priv->vlan >> 8) & 0xf; 5559 pktdata[hdrlen + 15] = priv->vlan & 0xff; 5560 pktlen += TAG_SZ; 5561 } 5562 } 5563 } 5564 5565 /* Pad packet if needed */ 5566 taglen = 0; 5567 tpid = PKT_U16_GET(pktdata, hdrlen + 12); 5568 if (tpid == 0x8100) { 5569 taglen = 4; 5570 } 5571 if (pktlen < (60 + taglen + hdrlen)) { 5572 pktlen = (60 + taglen + hdrlen); 5573 if (SKB_PADTO(skb, pktlen) != 0) { 5574 DBG_WARN(("Tx drop: skb_padto failed\n")); 5575 priv->stats.tx_dropped++; 5576 sinfo->tx.pkts_d_pad_fail++; 5577 dev_kfree_skb_any(skb); 5578 spin_unlock_irqrestore(&sinfo->lock, flags); 5579 return 0; 5580 } 5581 /* skb_padto may update the skb->data pointer */ 5582 pktdata = &skb->data[rcpulen]; 5583 } 5584 5585 if ((pktlen + FCS_SZ) > SOC_DCB_KNET_COUNT_MASK) { 5586 DBG_WARN(("Tx drop: size of pkt (%d) is out of range(%d)\n", 5587 (pktlen + FCS_SZ), SOC_DCB_KNET_COUNT_MASK)); 5588 sinfo->tx.pkts_d_over_limit++; 5589 priv->stats.tx_dropped++; 5590 dev_kfree_skb_any(skb); 5591 spin_unlock_irqrestore(&sinfo->lock, flags); 5592 return 0; 5593 } 5594 5595 dcb = desc->dcb_mem; 5596 meta = (sinfo->cmic_type == 'x') ? (uint32_t *)pktdata : dcb; 5597 memset(dcb, 0, sinfo->dcb_wsize * sizeof(uint32_t)); 5598 if (priv->flags & KCOM_NETIF_F_RCPU_ENCAP) { 5599 if (device_is_sand(sinfo)) { 5600 if (sinfo->cmic_type == 'x') { 5601 dcb[2] |= 1 << 19; 5602 /* Given Module Header exists and set first byte to be CPU channel */ 5603 pktdata[0] = cpu_channel; 5604 } else { 5605 dcb[1] |= 1 << 19; 5606 /* Set CPU channel */ 5607 dcb[2] = (cpu_channel & 0xff) << 24; 5608 } 5609 5610 } else if (sop != 0) { 5611 /* If module header SOP is non-zero, use RCPU meta data */ 5612 if (sinfo->cmic_type == 'x') { 5613 dcb[2] |= 1 << 19; 5614 } else { 5615 metalen = (sinfo->dcb_wsize - 3) * sizeof(uint32_t); 5616 if (metalen > RCPU_TX_META_SIZE) { 5617 metalen = RCPU_TX_META_SIZE; 5618 } 5619 metadata = (uint32_t *)&skb->data[RCPU_HDR_SIZE]; 5620 for (idx = 0; idx < BYTES2WORDS(metalen); idx++) { 5621 dcb[idx + 2] = ntohl(metadata[idx]); 5622 } 5623 dcb[1] |= 1 << 19; 5624 } 5625 } 5626 } else if (priv->port >= 0) { 5627 /* Send to physical port */ 5628 if (sinfo->cmic_type == 'x') { 5629 dcb[2] |= 1 << 19; 5630 } else { 5631 dcb[1] |= 1 << 19; 5632 } 5633 switch (sinfo->dcb_type) { 5634 case 23: 5635 case 26: 5636 case 30: 5637 case 31: 5638 case 34: 5639 case 37: 5640 dcb[2] = 0x81000000; 5641 dcb[3] = priv->port; 5642 dcb[3] |= (priv->qnum & 0xc00) << 20; 5643 dcb[4] = 0x00040000; 5644 dcb[4] |= (priv->qnum & 0x3ff) << 8; 5645 break; 5646 case 24: 5647 dcb[2] = 0xff000000; 5648 dcb[3] = 0x00000100; 5649 dcb[4] = priv->port; 5650 dcb[4] |= (priv->qnum & 0xfff) << 14; 5651 break; 5652 case 28: 5653 /* 5654 * If KCOM_NETIF_T_PORT, add PTCH+ITMH header 5655 * If KCOM_NETIF_T_VLAN, add PTCH+header 5656 */ 5657 pktdata = skb->data; 5658 memcpy(&pktdata[0], priv->system_headers, priv->system_headers_size); 5659 /* Set CPU channel */ 5660 dcb[2] = ((priv->qnum & 0xff) << 24); 5661 break; 5662 case 29: 5663 dcb[2] = 0x81000000; 5664 dcb[3] = priv->port; 5665 dcb[4] = 0x00100000; 5666 dcb[4] |= (priv->qnum & 0xfff) << 8; 5667 break; 5668 case 32: 5669 dcb[2] = 0x81000000; 5670 dcb[3] = priv->port; 5671 dcb[4] = 0x00004000; 5672 dcb[4] |= (priv->qnum & 0x3f) << 8; 5673 break; 5674 case 33: 5675 dcb[2] = 0x81000000; 5676 dcb[3] = (priv->port) << 2; 5677 dcb[4] = 0x00100000; 5678 dcb[4] |= (priv->qnum & 0xfff) << 8; 5679 break; 5680 case 35: 5681 dcb[2] = 0x81000000; 5682 dcb[3] = (priv->port) << 4; 5683 dcb[4] = 0x00400000; 5684 dcb[4] |= (priv->qnum & 0x3fff) << 8; 5685 break; 5686 case 36: 5687 if (sinfo->cmic_type == 'x') { 5688 meta[0] = htonl(0x81000000); 5689 meta[1] = htonl(priv->port); 5690 meta[2] = htonl(0x00008000 | (priv->qnum & 0x3f) << 9); 5691 } else { 5692 dcb[2] = 0x81000000; 5693 dcb[3] = priv->port; 5694 dcb[4] = 0x00008000; 5695 dcb[4] |= (priv->qnum & 0x3f) << 9; 5696 } 5697 break; 5698 case 38: 5699 if (sinfo->cmic_type == 'x') { 5700 meta[0] = htonl(0x81000000); 5701 meta[1] = htonl(priv->port); 5702 meta[2] = htonl(0x00004000 | (priv->qnum & 0x3f) << 8); 5703 } else { 5704 dcb[2] = 0x81000000; 5705 dcb[3] = priv->port; 5706 dcb[4] = 0x00004000; 5707 dcb[4] |= (priv->qnum & 0x3f) << 8; 5708 } 5709 break; 5710 case 39: 5711 if (device_is_dnx(sinfo)) { 5712 /* 5713 * if KCOM_NETIF_T_PORT, add MH+PTCH+ITMH header 5714 * if KCOM_NETIF_T_VLAN, add MH+PTCH+header 5715 */ 5716 pktdata = skb->data; 5717 memcpy(&pktdata[0], priv->system_headers, priv->system_headers_size); 5718 } 5719 break; 5720 case 40: 5721 if (sinfo->cmic_type == 'x') { 5722 meta[0] = htonl(0x81000000); 5723 meta[1] = htonl(priv->port | (priv->qnum & 0xc00) << 20); 5724 meta[2] = htonl(0x00040000 | (priv->qnum & 0x3ff) << 8); 5725 } else { 5726 dcb[2] = 0x81000000; 5727 dcb[3] = priv->port; 5728 dcb[3] |= (priv->qnum & 0xc00) << 20; 5729 dcb[4] = 0x00040000; 5730 dcb[4] |= (priv->qnum & 0x3ff) << 8; 5731 } 5732 break; 5733 default: 5734 dcb[2] = 0xff000000; 5735 dcb[3] = 0x00000100; 5736 dcb[4] = priv->port; 5737 break; 5738 } 5739 } 5740 5741 /* Optional SKB updates */ 5742 if (knet_tx_cb != NULL) { 5743 KNET_SKB_CB(skb)->netif_user_data = priv->cb_user_data; 5744 KNET_SKB_CB(skb)->dcb_type = sinfo->dcb_type & 0xFFFF; 5745 skb = knet_tx_cb(skb, sinfo->dev_no, meta); 5746 if (skb == NULL) { 5747 /* Consumed by call-back */ 5748 DBG_WARN(("Tx drop: Consumed by call-back\n")); 5749 priv->stats.tx_dropped++; 5750 sinfo->tx.pkts_d_callback++; 5751 spin_unlock_irqrestore(&sinfo->lock, flags); 5752 return 0; 5753 } 5754 /* Restore (possibly) altered packet variables 5755 * bit0 -bit15 of dcb[1] is used to save requested byte count 5756 */ 5757 if ((skb->len + FCS_SZ) <= SOC_DCB_KNET_COUNT_MASK) { 5758 pktlen = skb->len; 5759 if (pktlen < (60 + taglen + hdrlen)) { 5760 pktlen = (60 + taglen + hdrlen); 5761 if (SKB_PADTO(skb, pktlen) != 0) { 5762 DBG_WARN(("Tx drop: skb_padto failed\n")); 5763 priv->stats.tx_dropped++; 5764 sinfo->tx.pkts_d_pad_fail++; 5765 dev_kfree_skb_any(skb); 5766 spin_unlock_irqrestore(&sinfo->lock, flags); 5767 return 0; 5768 } 5769 DBG_SKB(("Packet padded to %d bytes after tx callback\n", pktlen)); 5770 } 5771 pktdata = skb->data; 5772 if (sinfo->cmic_type == 'x') { 5773 meta = (uint32_t *)pktdata; 5774 } 5775 } else { 5776 DBG_WARN(("Tx drop: size of pkt (%d) is out of range(%d)\n", 5777 (pktlen + FCS_SZ), SOC_DCB_KNET_COUNT_MASK)); 5778 sinfo->tx.pkts_d_over_limit++; 5779 priv->stats.tx_dropped++; 5780 sinfo->tx.pkts_d_callback++; 5781 dev_kfree_skb_any(skb); 5782 spin_unlock_irqrestore(&sinfo->lock, flags); 5783 return 0; 5784 } 5785 } 5786 5787 /* Do Tx timestamping */ 5788 if (bkn_skb_tx_flags(skb) & SKBTX_HW_TSTAMP) { 5789 KNET_SKB_CB(skb)->hwts = priv->tx_hwts; 5790 if ((priv->port >= 0) && (priv->tx_hwts & HWTSTAMP_TX_ON)) { 5791 /* TwoStep Processing of ptp-packets */ 5792 KNET_SKB_CB(skb)->port = priv->phys_port; 5793 bkn_hw_tstamp_tx_config(sinfo, priv->tx_hwts, PKT_TX_HDR_SIZE, skb, meta); 5794 5795 bkn_skb_tx_flags(skb) |= SKBTX_IN_PROGRESS; 5796 bkn_skb_tx_timestamp(skb); 5797 5798 } else if (priv->tx_hwts & HWTSTAMP_TX_ONESTEP_SYNC) { 5799 5800 /* OneStep Processing of ptp-packets */ 5801 KNET_SKB_CB(skb)->port = priv->phys_port; 5802 KNET_SKB_CB(skb)->ts = 0; 5803 bkn_hw_tstamp_tx_config(sinfo, priv->tx_hwts, PKT_TX_HDR_SIZE, skb, 5804 ((priv->port >= 0) ? meta : NULL)); 5805 5806 if (KNET_SKB_CB(skb)->ts != 0) { 5807 bkn_skb_tx_flags(skb) |= SKBTX_IN_PROGRESS; 5808 bkn_skb_tx_timestamp(skb); 5809 } 5810 5811 } 5812 } 5813 5814 /* Prepare for DMA */ 5815 desc->skb = skb; 5816 /* 5817 * Add FCS bytes 5818 * FCS bytes are always appended to packet by MAC on Dune devices 5819 */ 5820 if (!device_is_sand(sinfo)) { 5821 pktlen = pktlen + FCS_SZ; 5822 } 5823 desc->dma_size = pktlen; 5824 desc->skb_dma = BKN_DMA_MAP_SINGLE(sinfo->dma_dev, 5825 pktdata, desc->dma_size, 5826 BKN_DMA_TODEV); 5827 if (BKN_DMA_MAPPING_ERROR(sinfo->dma_dev, desc->skb_dma)) { 5828 priv->stats.tx_dropped++; 5829 dev_kfree_skb_any(skb); 5830 spin_unlock_irqrestore(&sinfo->lock, flags); 5831 return 0; 5832 } 5833 dcb[0] = desc->skb_dma; 5834 if (sinfo->cmic_type == 'x') { 5835 dcb[1] = DMA_TO_BUS_HI(desc->skb_dma >> 32); 5836 dcb[2] &= ~SOC_DCB_KNET_COUNT_MASK; 5837 dcb[2] |= pktlen; 5838 } else { 5839 dcb[1] &= ~SOC_DCB_KNET_COUNT_MASK; 5840 dcb[1] |= pktlen; 5841 } 5842 5843 bkn_dump_dcb("Tx RCPU", dcb, sinfo->dcb_wsize, XGS_DMA_TX_CHAN); 5844 DBG_DCB_TX(("Add Tx DCB @ 0x%08x (%d) [%d free] (%d bytes).\n", 5845 (uint32_t)desc->dcb_dma, sinfo->tx.cur, 5846 sinfo->tx.free, pktlen)); 5847 bkn_dump_pkt(pktdata, pktlen, XGS_DMA_TX_CHAN); 5848 5849 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN)) { 5850 if (sinfo->cmic_type == 'x') { 5851 dcb[2] |= 1 << 24 | 1 << 16; 5852 } else { 5853 dcb[1] |= 1 << 24 | 1 << 16; 5854 } 5855 } else { 5856 bkn_tx_dma_start(sinfo); 5857 } 5858 if (++sinfo->tx.cur >= MAX_TX_DCBS) { 5859 sinfo->tx.cur = 0; 5860 } 5861 sinfo->tx.free--; 5862 5863 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN) && !sinfo->tx.api_active) { 5864 /* DMA run to the new halt location */ 5865 bkn_cdma_goto(sinfo, XGS_DMA_TX_CHAN, 5866 sinfo->tx.desc[sinfo->tx.cur].dcb_dma); 5867 } 5868 5869 priv->stats.tx_packets++; 5870 priv->stats.tx_bytes += pktlen; 5871 sinfo->tx.pkts++; 5872 } else { 5873 DBG_WARN(("Tx drop: No DMA resources\n")); 5874 priv->stats.tx_dropped++; 5875 sinfo->tx.pkts_d_dma_resrc++; 5876 dev_kfree_skb_any(skb); 5877 } 5878 5879 /* Check our Tx resources */ 5880 if (sinfo->tx.free <= 1) { 5881 bkn_suspend_tx(sinfo); 5882 } 5883 5884 NETDEV_UPDATE_TRANS_START_TIME(dev); 5885 5886 spin_unlock_irqrestore(&sinfo->lock, flags); 5887 5888 return 0; 5889 } 5890 5891 static void 5892 bkn_timer_func(bkn_switch_info_t *sinfo) 5893 { 5894 unsigned long flags; 5895 int chan; 5896 int restart_timer; 5897 5898 spin_lock_irqsave(&sinfo->lock, flags); 5899 5900 sinfo->timer_runs++; 5901 5902 restart_timer = 0; 5903 for (chan = 0; chan < sinfo->rx_chans; chan++) { 5904 /* Restart channel if not running */ 5905 if (sinfo->rx[chan].running == 0) { 5906 bkn_rx_refill(sinfo, chan); 5907 if (bkn_rx_restart(sinfo, chan) != 0) { 5908 restart_timer = 1; 5909 } 5910 } 5911 } 5912 5913 if (restart_timer) { 5914 /* Presumably still out of memory */ 5915 sinfo->timer.expires = jiffies + 1; 5916 add_timer(&sinfo->timer); 5917 } else { 5918 sinfo->timer_queued = 0; 5919 } 5920 5921 spin_unlock_irqrestore(&sinfo->lock, flags); 5922 } 5923 5924 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4,15,0)) 5925 static void 5926 bkn_timer(unsigned long context) 5927 { 5928 bkn_switch_info_t *sinfo = (bkn_switch_info_t *)context; 5929 return bkn_timer_func(sinfo); 5930 } 5931 #else 5932 static void 5933 bkn_timer(struct timer_list *t) 5934 { 5935 bkn_switch_info_t *sinfo = from_timer(sinfo, t, timer); 5936 return bkn_timer_func(sinfo); 5937 } 5938 #endif 5939 5940 static void 5941 bkn_rx_add_tokens(bkn_switch_info_t *sinfo, int chan) 5942 { 5943 unsigned long cur_jif, ticks; 5944 uint32_t tokens_per_tick; 5945 bkn_desc_info_t *desc; 5946 5947 tokens_per_tick = sinfo->rx[chan].rate_max / HZ; 5948 cur_jif = jiffies; 5949 ticks = cur_jif - sinfo->rx[chan].tok_jif; 5950 sinfo->rx[chan].tokens += ticks * tokens_per_tick; 5951 sinfo->rx[chan].tok_jif = cur_jif; 5952 if (sinfo->rx[chan].tokens > sinfo->rx[chan].burst_max) { 5953 sinfo->rx[chan].tokens = sinfo->rx[chan].burst_max; 5954 } 5955 5956 /* Restart channel if Rx is suppressed */ 5957 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 5958 bkn_rx_refill(sinfo, chan); 5959 desc = &sinfo->rx[chan].desc[sinfo->rx[chan].dirty]; 5960 if (desc->dcb_dma == sinfo->halt_addr[XGS_DMA_RX_CHAN + chan]) { 5961 desc = &sinfo->rx[chan].desc[MAX_RX_DCBS]; 5962 bkn_cdma_goto(sinfo, XGS_DMA_RX_CHAN + chan, desc->dcb_dma); 5963 } 5964 } else { 5965 if (sinfo->rx[chan].running == 0) { 5966 bkn_rx_refill(sinfo, chan); 5967 bkn_rx_restart(sinfo, chan); 5968 } 5969 } 5970 } 5971 5972 static void 5973 bkn_rxtick_func(bkn_switch_info_t *sinfo) 5974 { 5975 unsigned long flags; 5976 unsigned long cur_jif, ticks; 5977 uint32_t pkt_diff; 5978 int chan; 5979 5980 spin_lock_irqsave(&sinfo->lock, flags); 5981 5982 sinfo->rxtick.expires = jiffies + sinfo->rxtick_jiffies; 5983 5984 /* For debug purposes we maintain a rough actual packet rate */ 5985 if (++sinfo->rxticks >= sinfo->rxticks_per_sec) { 5986 for (chan = 0; chan < sinfo->rx_chans; chan++) { 5987 pkt_diff = sinfo->rx[chan].pkts - sinfo->rx[chan].pkts_ref; 5988 cur_jif = jiffies; 5989 ticks = cur_jif - sinfo->rx[chan].rate_jif; 5990 sinfo->rx[chan].rate = (pkt_diff * HZ) / ticks; 5991 sinfo->rx[chan].rate_jif = cur_jif; 5992 sinfo->rx[chan].pkts_ref = sinfo->rx[chan].pkts; 5993 } 5994 sinfo->rxticks = 0; 5995 } 5996 5997 /* Update tokens for Rx rate control */ 5998 for (chan = 0; chan < sinfo->rx_chans; chan++) { 5999 if (sinfo->rx[chan].tokens < sinfo->rx[chan].burst_max) { 6000 bkn_rx_add_tokens(sinfo, chan); 6001 } 6002 } 6003 6004 spin_unlock_irqrestore(&sinfo->lock, flags); 6005 6006 add_timer(&sinfo->rxtick); 6007 } 6008 6009 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4,15,0)) 6010 static void 6011 bkn_rxtick(unsigned long context) 6012 { 6013 bkn_switch_info_t *sinfo = (bkn_switch_info_t *)context; 6014 return bkn_rxtick_func(sinfo); 6015 } 6016 #else 6017 static void 6018 bkn_rxtick(struct timer_list *t) 6019 { 6020 bkn_switch_info_t *sinfo = from_timer(sinfo, t, rxtick); 6021 return bkn_rxtick_func(sinfo); 6022 } 6023 #endif 6024 6025 static void 6026 bkn_rx_rate_config(bkn_switch_info_t *sinfo) 6027 { 6028 unsigned long flags; 6029 int chan; 6030 uint32_t rxticks_per_sec, rps; 6031 uint32_t jiffies_per_rxtick; 6032 uint32_t tokens_per_rxtick; 6033 6034 spin_lock_irqsave(&sinfo->lock, flags); 6035 6036 /* Calculate the minimum update frequency across all channels */ 6037 rxticks_per_sec = 1; 6038 for (chan = 0; chan < NUM_RX_CHAN; chan++) { 6039 if (sinfo->rx[chan].burst_max == 0) { 6040 sinfo->rx[chan].burst_max = sinfo->rx[chan].rate_max / 10; 6041 } 6042 rps = sinfo->rx[chan].rate_max / sinfo->rx[chan].burst_max; 6043 if (rxticks_per_sec < rps) { 6044 rxticks_per_sec = rps; 6045 } 6046 } 6047 6048 /* Convert update frequency to system ticks */ 6049 jiffies_per_rxtick = HZ / rxticks_per_sec; 6050 if (jiffies_per_rxtick == 0) { 6051 jiffies_per_rxtick = 1; 6052 } 6053 rxticks_per_sec = HZ / jiffies_per_rxtick; 6054 6055 for (chan = 0; chan < NUM_RX_CHAN; chan++) { 6056 /* Ensure that burst size satifies overall rate */ 6057 tokens_per_rxtick = sinfo->rx[chan].rate_max / rxticks_per_sec; 6058 if (sinfo->rx[chan].burst_max < tokens_per_rxtick) { 6059 sinfo->rx[chan].burst_max = tokens_per_rxtick; 6060 } 6061 /* Ensure that rate has a sane value */ 6062 if (sinfo->rx[chan].rate_max != 0) { 6063 if (sinfo->rx[chan].rate_max < rxticks_per_sec) { 6064 sinfo->rx[chan].rate_max = rxticks_per_sec; 6065 } 6066 } 6067 sinfo->rx[chan].tokens = sinfo->rx[chan].burst_max; 6068 } 6069 6070 /* Update timer controls */ 6071 sinfo->rxticks_per_sec = rxticks_per_sec; 6072 sinfo->rxtick_jiffies = jiffies_per_rxtick; 6073 6074 spin_unlock_irqrestore(&sinfo->lock, flags); 6075 } 6076 6077 static void 6078 bkn_destroy_sinfo(bkn_switch_info_t *sinfo) 6079 { 6080 list_del(&sinfo->list); 6081 bkn_free_dcbs(sinfo); 6082 kfree(sinfo); 6083 } 6084 6085 static bkn_switch_info_t * 6086 bkn_create_sinfo(int dev_no) 6087 { 6088 bkn_switch_info_t *sinfo; 6089 int chan; 6090 6091 if ((sinfo = kmalloc(sizeof(*sinfo), GFP_KERNEL)) == NULL) { 6092 return NULL; 6093 } 6094 memset(sinfo, 0, sizeof(*sinfo)); 6095 INIT_LIST_HEAD(&sinfo->ndev_list); 6096 INIT_LIST_HEAD(&sinfo->rxpf_list); 6097 sinfo->base_addr = lkbde_get_dev_virt(dev_no); 6098 sinfo->dma_dev = lkbde_get_dma_dev(dev_no); 6099 sinfo->pdev = lkbde_get_hw_dev(dev_no); 6100 sinfo->dev_no = dev_no; 6101 sinfo->evt_idx = -1; 6102 6103 spin_lock_init(&sinfo->lock); 6104 skb_queue_head_init(&sinfo->tx_ptp_queue); 6105 INIT_WORK(&sinfo->tx_ptp_work, bkn_hw_tstamp_tx_work); 6106 6107 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4,15,0)) 6108 init_timer(&sinfo->timer); 6109 sinfo->timer.data = (unsigned long)sinfo; 6110 sinfo->timer.function = bkn_timer; 6111 #else 6112 timer_setup(&sinfo->timer, bkn_timer, 0); 6113 #endif 6114 sinfo->timer.expires = jiffies + 1; 6115 6116 INIT_LIST_HEAD(&sinfo->tx.api_dcb_list); 6117 6118 for (chan = 0; chan < NUM_RX_CHAN; chan++) { 6119 INIT_LIST_HEAD(&sinfo->rx[chan].api_dcb_list); 6120 sinfo->rx[chan].use_rx_skb = use_rx_skb; 6121 } 6122 6123 /* 6124 * Check for dual DMA mode where Rx DMA channel 0 uses DMA buffers 6125 * provided by the BCM API, and the remaining Rx DMA channel(s) 6126 * use socket buffers (SKB) provided by the Linux kernel. 6127 */ 6128 if (use_rx_skb == 2) { 6129 sinfo->rx[0].use_rx_skb = 0; 6130 } 6131 6132 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4,15,0)) 6133 init_timer(&sinfo->rxtick); 6134 sinfo->rxtick.data = (unsigned long)sinfo; 6135 sinfo->rxtick.function = bkn_rxtick; 6136 #else 6137 timer_setup(&sinfo->rxtick, bkn_rxtick, 0); 6138 #endif 6139 sinfo->rxtick.expires = jiffies + 1; 6140 6141 for (chan = 0; chan < NUM_RX_CHAN; chan++) { 6142 sinfo->rx[chan].rate_max = rx_rate[chan]; 6143 sinfo->rx[chan].burst_max = rx_burst[chan]; 6144 } 6145 bkn_rx_rate_config(sinfo); 6146 6147 add_timer(&sinfo->rxtick); 6148 6149 list_add_tail(&sinfo->list, &_sinfo_list); 6150 6151 return sinfo; 6152 } 6153 6154 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29)) 6155 static const struct net_device_ops bkn_netdev_ops = { 6156 .ndo_open = bkn_open, 6157 .ndo_stop = bkn_stop, 6158 .ndo_start_xmit = bkn_tx, 6159 .ndo_get_stats = bkn_get_stats, 6160 .ndo_validate_addr = eth_validate_addr, 6161 .ndo_set_rx_mode = bkn_set_multicast_list, 6162 .ndo_set_mac_address = bkn_set_mac_address, 6163 .ndo_do_ioctl = bkn_ioctl, 6164 .ndo_tx_timeout = NULL, 6165 .ndo_change_mtu = bkn_change_mtu, 6166 #ifdef CONFIG_NET_POLL_CONTROLLER 6167 .ndo_poll_controller = bkn_poll_controller, 6168 #endif 6169 }; 6170 #endif 6171 6172 static void 6173 bkn_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) 6174 { 6175 strlcpy(drvinfo->driver, "bcm-knet", sizeof(drvinfo->driver)); 6176 snprintf(drvinfo->version, sizeof(drvinfo->version), "%d", KCOM_VERSION); 6177 strlcpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version)); 6178 strlcpy(drvinfo->bus_info, "N/A", sizeof(drvinfo->bus_info)); 6179 } 6180 6181 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0)) 6182 static int 6183 bkn_get_ts_info(struct net_device *dev, struct ethtool_ts_info *info) 6184 { 6185 bkn_priv_t *priv = netdev_priv(dev); 6186 bkn_switch_info_t *sinfo = priv->sinfo; 6187 6188 switch (sinfo->dcb_type) { 6189 case 26: 6190 case 32: 6191 case 33: 6192 case 36: 6193 case 38: 6194 case 40: 6195 info->so_timestamping = SOF_TIMESTAMPING_TX_HARDWARE | 6196 SOF_TIMESTAMPING_TX_SOFTWARE | 6197 SOF_TIMESTAMPING_RX_HARDWARE | 6198 SOF_TIMESTAMPING_RX_SOFTWARE | 6199 SOF_TIMESTAMPING_SOFTWARE | 6200 SOF_TIMESTAMPING_RAW_HARDWARE; 6201 info->tx_types = 1 << HWTSTAMP_TX_OFF | 1 << HWTSTAMP_TX_ON; 6202 info->rx_filters = 1 << HWTSTAMP_FILTER_NONE | 1 << HWTSTAMP_FILTER_ALL; 6203 if (knet_hw_tstamp_ptp_clock_index_cb) { 6204 info->phc_index = knet_hw_tstamp_ptp_clock_index_cb(sinfo->dev_no); 6205 } else { 6206 info->phc_index = -1; 6207 } 6208 break; 6209 default: 6210 info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | 6211 SOF_TIMESTAMPING_RX_SOFTWARE | 6212 SOF_TIMESTAMPING_SOFTWARE; 6213 info->phc_index = -1; 6214 break; 6215 } 6216 6217 return 0; 6218 } 6219 #endif 6220 6221 static const struct ethtool_ops bkn_ethtool_ops = { 6222 .get_drvinfo = bkn_get_drvinfo, 6223 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0)) 6224 .get_ts_info = bkn_get_ts_info, 6225 #endif 6226 }; 6227 6228 static struct net_device * 6229 bkn_init_ndev(u8 *mac, char *name) 6230 { 6231 struct net_device *dev; 6232 6233 /* Create Ethernet device */ 6234 dev = alloc_etherdev(sizeof(bkn_priv_t)); 6235 6236 if (dev == NULL) { 6237 DBG_WARN(("Error allocating Ethernet device.\n")); 6238 return NULL; 6239 } 6240 #ifdef SET_MODULE_OWNER 6241 SET_MODULE_OWNER(dev); 6242 #endif 6243 6244 /* Set the device MAC address */ 6245 memcpy(dev->dev_addr, mac, 6); 6246 6247 /* Device information -- not available right now */ 6248 dev->irq = 0; 6249 dev->base_addr = 0; 6250 6251 /* Default MTU should not exceed MTU of switch front-panel ports */ 6252 dev->mtu = default_mtu; 6253 if (dev->mtu == 0) { 6254 dev->mtu = rx_buffer_size; 6255 } 6256 6257 /* Device vectors */ 6258 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29)) 6259 dev->netdev_ops = &bkn_netdev_ops; 6260 #else 6261 dev->open = bkn_open; 6262 dev->hard_start_xmit = bkn_tx; 6263 dev->stop = bkn_stop; 6264 dev->set_multicast_list = bkn_set_multicast_list; 6265 dev->do_ioctl = NULL; 6266 dev->get_stats = bkn_get_stats; 6267 dev->change_mtu = bkn_change_mtu; 6268 #ifdef CONFIG_NET_POLL_CONTROLLER 6269 dev->poll_controller = bkn_poll_controller; 6270 #endif 6271 #endif 6272 dev->ethtool_ops = &bkn_ethtool_ops; 6273 if (name && *name) { 6274 strncpy(dev->name, name, IFNAMSIZ-1); 6275 } 6276 6277 bkn_dev_net_set(dev, current->nsproxy->net_ns); 6278 6279 /* Register the kernel Ethernet device */ 6280 if (register_netdev(dev)) { 6281 DBG_WARN(("Error registering Ethernet device.\n")); 6282 free_netdev(dev); 6283 return NULL; 6284 } 6285 DBG_VERB(("Created Ethernet device %s.\n", dev->name)); 6286 6287 return dev; 6288 } 6289 6290 /* 6291 * Device Link Control Proc Read Entry 6292 */ 6293 static int 6294 bkn_proc_link_show(struct seq_file *m, void *v) 6295 { 6296 struct list_head *slist, *dlist; 6297 struct net_device *dev; 6298 bkn_priv_t *priv; 6299 bkn_switch_info_t *sinfo; 6300 unsigned long flags; 6301 6302 seq_printf(m, "Software link status:\n"); 6303 list_for_each(slist, &_sinfo_list) { 6304 sinfo = (bkn_switch_info_t *)slist; 6305 spin_lock_irqsave(&sinfo->lock, flags); 6306 list_for_each(dlist, &sinfo->ndev_list) { 6307 priv = (bkn_priv_t *)dlist; 6308 dev = priv->dev; 6309 if (dev) { 6310 seq_printf(m, " %-14s %s\n", dev->name, 6311 netif_carrier_ok(dev) ? "up" : "down"); 6312 } 6313 } 6314 spin_unlock_irqrestore(&sinfo->lock, flags); 6315 } 6316 return 0; 6317 } 6318 6319 static int 6320 bkn_proc_link_open(struct inode * inode, struct file * file) 6321 { 6322 return single_open(file, bkn_proc_link_show, NULL); 6323 } 6324 6325 /* 6326 * Device Link Control Proc Write Entry 6327 * 6328 * Syntax: 6329 * <netif>=up|down 6330 * 6331 * Where <netif> is a virtual network interface name. 6332 * 6333 * Examples: 6334 * eth4=up 6335 * eth4=down 6336 */ 6337 static ssize_t 6338 bkn_proc_link_write(struct file *file, const char *buf, 6339 size_t count, loff_t *loff) 6340 { 6341 struct list_head *slist, *dlist; 6342 struct net_device *dev; 6343 bkn_priv_t *priv; 6344 bkn_switch_info_t *sinfo; 6345 unsigned long flags; 6346 char link_str[40]; 6347 char *ptr; 6348 char *newline; 6349 6350 if (count > sizeof(link_str)) { 6351 count = sizeof(link_str) - 1; 6352 link_str[count] = '\0'; 6353 } 6354 if (copy_from_user(link_str, buf, count)) { 6355 return -EFAULT; 6356 } 6357 link_str[count] = 0; 6358 newline = strchr(link_str, '\n'); 6359 if (newline) { 6360 /* Chop off the trailing newline */ 6361 *newline = '\0'; 6362 } 6363 6364 if ((ptr = strchr(link_str, '=')) == NULL && 6365 (ptr = strchr(link_str, ':')) == NULL) { 6366 gprintk("Error: link syntax not recognized: '%s'\n", link_str); 6367 return count; 6368 } 6369 *ptr++ = 0; 6370 6371 dev = NULL; 6372 list_for_each(slist, &_sinfo_list) { 6373 sinfo = (bkn_switch_info_t *)slist; 6374 spin_lock_irqsave(&sinfo->lock, flags); 6375 list_for_each(dlist, &sinfo->ndev_list) { 6376 priv = (bkn_priv_t *)dlist; 6377 if (priv->dev) { 6378 if (strcmp(priv->dev->name, link_str) == 0) { 6379 dev = priv->dev; 6380 break; 6381 } 6382 } 6383 } 6384 if (dev) { 6385 if (strcmp(ptr, "up") == 0) { 6386 netif_carrier_on(dev); 6387 } else if (strcmp(ptr, "down") == 0) { 6388 netif_carrier_off(dev); 6389 } else { 6390 gprintk("Warning: unknown link state setting: '%s'\n", ptr); 6391 } 6392 spin_unlock_irqrestore(&sinfo->lock, flags); 6393 return count; 6394 } 6395 spin_unlock_irqrestore(&sinfo->lock, flags); 6396 } 6397 6398 gprintk("Warning: unknown network interface: '%s'\n", link_str); 6399 6400 return count; 6401 } 6402 6403 struct file_operations bkn_proc_link_file_ops = { 6404 owner: THIS_MODULE, 6405 open: bkn_proc_link_open, 6406 read: seq_read, 6407 llseek: seq_lseek, 6408 write: bkn_proc_link_write, 6409 release: single_release, 6410 }; 6411 6412 /* 6413 * Device Rate Control Proc Read Entry 6414 */ 6415 static int 6416 bkn_proc_rate_show(struct seq_file *m, void *v) 6417 { 6418 int unit = 0; 6419 struct list_head *list; 6420 bkn_switch_info_t *sinfo; 6421 int chan; 6422 6423 list_for_each(list, &_sinfo_list) { 6424 sinfo = (bkn_switch_info_t *)list; 6425 6426 seq_printf(m, "Rate control (unit %d):\n", unit); 6427 for (chan = 0; chan < sinfo->rx_chans; chan++) { 6428 seq_printf(m, " Rx%d max rate %8u\n", 6429 chan, sinfo->rx[chan].rate_max); 6430 seq_printf(m, " Rx%d max burst %8u\n", 6431 chan, sinfo->rx[chan].burst_max); 6432 seq_printf(m, " Rx%d rate %8u\n", 6433 chan, sinfo->rx[chan].rate); 6434 seq_printf(m, " Rx%d tokens %8u\n", 6435 chan, sinfo->rx[chan].tokens); 6436 } 6437 6438 unit++; 6439 } 6440 return 0; 6441 } 6442 6443 static int 6444 bkn_proc_rate_open(struct inode * inode, struct file * file) 6445 { 6446 return single_open(file, bkn_proc_rate_show, NULL); 6447 } 6448 6449 /* 6450 * Device Rate Control Proc Write Entry 6451 * 6452 * Syntax: 6453 * [<unit>:]rx_rate=<rate0>[,<rate1>[,<rate2]] 6454 * 6455 * Where <rate0> is packets/sec for the first Rx DMA channel, 6456 * <rate1> is packets/sec for the second Rx DMA channel, etc. 6457 * 6458 * Examples: 6459 * rx_rate=5000 6460 * 0:rx_rate=10000,10000 6461 * 1:rx_rate=10000,5000 6462 */ 6463 static ssize_t 6464 bkn_proc_rate_write(struct file *file, const char *buf, 6465 size_t count, loff_t *loff) 6466 { 6467 bkn_switch_info_t *sinfo; 6468 char rate_str[80]; 6469 char *ptr; 6470 int unit, chan; 6471 6472 if (count > sizeof(rate_str)) { 6473 count = sizeof(rate_str) - 1; 6474 rate_str[count] = '\0'; 6475 } 6476 if (copy_from_user(rate_str, buf, count)) { 6477 return -EFAULT; 6478 } 6479 6480 unit = simple_strtol(rate_str, NULL, 10); 6481 sinfo = bkn_sinfo_from_unit(unit); 6482 if (sinfo == NULL) { 6483 gprintk("Warning: unknown unit: %d\n", unit); 6484 return count; 6485 } 6486 6487 if ((ptr = strstr(rate_str, "rx_rate=")) != NULL) { 6488 ptr += 7; 6489 chan = 0; 6490 do { 6491 ptr++; 6492 sinfo->rx[chan].rate_max = simple_strtol(ptr, NULL, 10); 6493 } while ((ptr = strchr(ptr, ',')) != NULL && ++chan < sinfo->rx_chans); 6494 bkn_rx_rate_config(sinfo); 6495 } else if ((ptr = strstr(rate_str, "rx_burst=")) != NULL) { 6496 ptr += 8; 6497 chan = 0; 6498 do { 6499 ptr++; 6500 sinfo->rx[chan].burst_max = simple_strtol(ptr, NULL, 10); 6501 } while ((ptr = strchr(ptr, ',')) != NULL && ++chan < sinfo->rx_chans); 6502 bkn_rx_rate_config(sinfo); 6503 } else { 6504 gprintk("Warning: unknown configuration setting\n"); 6505 } 6506 6507 return count; 6508 } 6509 6510 struct file_operations bkn_proc_rate_file_ops = { 6511 owner: THIS_MODULE, 6512 open: bkn_proc_rate_open, 6513 read: seq_read, 6514 llseek: seq_lseek, 6515 write: bkn_proc_rate_write, 6516 release: single_release, 6517 }; 6518 6519 /* 6520 * Driver DMA Proc Entry 6521 * 6522 * This output can be rather large (> PAGE_SIZE) so we use the 6523 * seq_file interface to do the output. Special header records 6524 * are indicated by a negative DCB index. 6525 */ 6526 typedef struct { 6527 int dev_no; /* Current unit */ 6528 int rx_dma; /* 0: Tx DMA, 1: Rx DMA*/ 6529 int ch_no; /* DMA channel no. (Rx only) */ 6530 int idx; /* DCB index */ 6531 } bkn_seq_dma_iter_t; 6532 6533 /* From current record, move forward 'pos' records */ 6534 static int 6535 bkn_seq_dma_next_pos(bkn_seq_dma_iter_t *iter, loff_t pos) 6536 { 6537 bkn_switch_info_t *sinfo; 6538 6539 sinfo = bkn_sinfo_from_unit(iter->dev_no); 6540 while (pos) { 6541 if (iter->rx_dma) { 6542 if (++iter->idx >= MAX_RX_DCBS + 1) { 6543 iter->idx = -1; 6544 if (++iter->ch_no >= sinfo->rx_chans) { 6545 iter->rx_dma = 0; 6546 iter->ch_no = 0; 6547 iter->dev_no++; 6548 if ((sinfo = bkn_sinfo_from_unit(iter->dev_no)) == NULL) { 6549 return -1; 6550 } 6551 } 6552 } 6553 } else { 6554 if (++iter->idx >= MAX_TX_DCBS + 1) { 6555 iter->idx = -1; 6556 iter->rx_dma = 1; 6557 } 6558 } 6559 pos--; 6560 } 6561 return 0; 6562 } 6563 6564 /* Initialize private data and move to requested start record */ 6565 static void * 6566 bkn_seq_dma_start(struct seq_file *s, loff_t *pos) 6567 { 6568 bkn_seq_dma_iter_t *iter; 6569 6570 iter = kmalloc(sizeof(bkn_seq_dma_iter_t), GFP_KERNEL); 6571 if (!iter) { 6572 return NULL; 6573 } 6574 memset(iter, 0, sizeof(*iter)); 6575 iter->idx = -2; 6576 if (bkn_seq_dma_next_pos(iter, *pos) < 0) { 6577 kfree(iter); 6578 return NULL; 6579 } 6580 return iter; 6581 } 6582 6583 /* Move to next record */ 6584 static void * 6585 bkn_seq_dma_next(struct seq_file *s, void *v, loff_t *pos) 6586 { 6587 bkn_seq_dma_iter_t *iter = (bkn_seq_dma_iter_t *)v; 6588 void *rv = iter; 6589 6590 if (bkn_seq_dma_next_pos(iter, 1) < 0) { 6591 return NULL; 6592 } 6593 (*pos)++; 6594 return rv; 6595 } 6596 6597 /* Release private data */ 6598 static void 6599 bkn_seq_dma_stop(struct seq_file *s, void *v) 6600 { 6601 if (v) { 6602 kfree(v); 6603 } 6604 } 6605 6606 /* Print current record */ 6607 static int 6608 bkn_seq_dma_show(struct seq_file *s, void *v) 6609 { 6610 bkn_seq_dma_iter_t *iter = (bkn_seq_dma_iter_t *)v; 6611 bkn_switch_info_t *sinfo; 6612 uint32_t *dcb = NULL; 6613 bkn_dcb_chain_t *dcb_chain = NULL; 6614 struct list_head *curr, *next; 6615 unsigned long flags; 6616 int chan, cnt; 6617 6618 sinfo = bkn_sinfo_from_unit(iter->dev_no); 6619 if (sinfo == NULL) { 6620 /* Should not happen */ 6621 return 0; 6622 } 6623 6624 if (iter->rx_dma == 0) { 6625 if (iter->idx == -2) { 6626 seq_printf(s, "Pending events: 0x%x\n", sinfo->dma_events); 6627 } else if (iter->idx == -1) { 6628 spin_lock_irqsave(&sinfo->lock, flags); 6629 curr = &sinfo->tx.api_dcb_list; 6630 dcb_chain = sinfo->tx.api_dcb_chain; 6631 while (dcb_chain) { 6632 seq_printf(s, " [0x%08lx]--->\n", (unsigned long)dcb_chain->dcb_dma); 6633 for (cnt = 0; cnt < dcb_chain->dcb_cnt; cnt++) { 6634 dcb = &dcb_chain->dcb_mem[sinfo->dcb_wsize * cnt]; 6635 if (sinfo->cmic_type == 'x') { 6636 seq_printf(s, " DCB %2d: 0x%08x 0x%08x 0x%08x 0x%08x\n", cnt, 6637 dcb[0], dcb[1], dcb[2], dcb[sinfo->dcb_wsize-1]); 6638 } else { 6639 seq_printf(s, " DCB %2d: 0x%08x 0x%08x ... 0x%08x\n", cnt, 6640 dcb[0], dcb[1], dcb[sinfo->dcb_wsize-1]); 6641 } 6642 } 6643 next = curr->next; 6644 if (next != &sinfo->tx.api_dcb_list) { 6645 dcb_chain = list_entry(next, bkn_dcb_chain_t, list); 6646 curr = next; 6647 } else { 6648 dcb_chain = NULL; 6649 } 6650 } 6651 dcb = NULL; 6652 spin_unlock_irqrestore(&sinfo->lock, flags); 6653 seq_printf(s, 6654 "Tx DCB info (unit %d):\n" 6655 " api: %d\n" 6656 " dirty: %d\n" 6657 " cur: %d\n" 6658 " free: %d\n" 6659 " pause: %s\n", 6660 iter->dev_no, 6661 sinfo->tx.api_active, 6662 sinfo->tx.dirty, 6663 sinfo->tx.cur, 6664 sinfo->tx.free, 6665 netif_queue_stopped(sinfo->dev) ? "yes" : "no"); 6666 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN)) { 6667 bkn_desc_info_t *desc = &sinfo->tx.desc[0]; 6668 uint64_t halt = sinfo->halt_addr[XGS_DMA_TX_CHAN]; 6669 seq_printf(s, 6670 " halt: %d\n", 6671 (int)((uint32_t)(halt - desc->dcb_dma) / (sinfo->dcb_wsize * sizeof(uint32_t)))); 6672 } 6673 } else { 6674 dcb = sinfo->tx.desc[iter->idx].dcb_mem; 6675 if (iter->idx == 0) { 6676 seq_printf(s, " [0x%08lx]--->\n", (unsigned long)sinfo->tx.desc[0].dcb_dma); 6677 } 6678 } 6679 } else { 6680 if (iter->idx == -1) { 6681 chan = iter->ch_no; 6682 seq_printf(s, 6683 "Rx%d DCB info (unit %d):\n" 6684 " api: %d\n" 6685 " dirty: %d\n" 6686 " cur: %d\n" 6687 " free: %d\n" 6688 " run: %d\n", 6689 chan, iter->dev_no, 6690 sinfo->rx[chan].api_active, 6691 sinfo->rx[chan].dirty, 6692 sinfo->rx[chan].cur, 6693 sinfo->rx[chan].free, 6694 sinfo->rx[chan].running); 6695 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan) && sinfo->rx[chan].use_rx_skb) { 6696 bkn_desc_info_t *desc = &sinfo->rx[chan].desc[0]; 6697 uint64_t halt = sinfo->halt_addr[XGS_DMA_RX_CHAN + chan]; 6698 seq_printf(s, 6699 " halt: %d\n", 6700 (int)((uint32_t)(halt - desc->dcb_dma) / (sinfo->dcb_wsize * sizeof(uint32_t)))); 6701 } 6702 } else if (sinfo->rx[iter->ch_no].use_rx_skb) { 6703 dcb = sinfo->rx[iter->ch_no].desc[iter->idx].dcb_mem; 6704 if (iter->idx == 0) { 6705 seq_printf(s, " [0x%08lx]--->\n", (unsigned long)sinfo->rx[iter->ch_no].desc[0].dcb_dma); 6706 } 6707 } else if (!sinfo->rx[iter->ch_no].use_rx_skb && iter->idx == 0) { 6708 spin_lock_irqsave(&sinfo->lock, flags); 6709 curr = &sinfo->rx[iter->ch_no].api_dcb_list; 6710 dcb_chain = sinfo->rx[iter->ch_no].api_dcb_chain; 6711 while (dcb_chain) { 6712 seq_printf(s, " [0x%08lx]--->\n", (unsigned long)dcb_chain->dcb_dma); 6713 for (cnt = 0; cnt < dcb_chain->dcb_cnt; cnt++) { 6714 dcb = &dcb_chain->dcb_mem[sinfo->dcb_wsize * cnt]; 6715 if (sinfo->cmic_type == 'x') { 6716 seq_printf(s, " DCB %2d: 0x%08x 0x%08x 0x%08x 0x%08x\n", cnt, 6717 dcb[0], dcb[1], dcb[2], dcb[sinfo->dcb_wsize-1]); 6718 } else { 6719 seq_printf(s, " DCB %2d: 0x%08x 0x%08x ... 0x%08x\n", cnt, 6720 dcb[0], dcb[1], dcb[sinfo->dcb_wsize-1]); 6721 } 6722 } 6723 next = curr->next; 6724 if (next != &sinfo->rx[iter->ch_no].api_dcb_list) { 6725 dcb_chain = list_entry(next, bkn_dcb_chain_t, list); 6726 curr = next; 6727 } else { 6728 dcb_chain = NULL; 6729 } 6730 } 6731 dcb = NULL; 6732 spin_unlock_irqrestore(&sinfo->lock, flags); 6733 } 6734 } 6735 if (dcb) { 6736 if (sinfo->cmic_type == 'x') { 6737 seq_printf(s, " DCB %2d: 0x%08x 0x%08x 0x%08x 0x%08x\n", iter->idx, 6738 dcb[0], dcb[1], dcb[2], dcb[sinfo->dcb_wsize-1]); 6739 } else { 6740 seq_printf(s, " DCB %2d: 0x%08x 0x%08x ... 0x%08x\n", iter->idx, 6741 dcb[0], dcb[1], dcb[sinfo->dcb_wsize-1]); 6742 } 6743 } 6744 return 0; 6745 } 6746 6747 static struct seq_operations bkn_seq_dma_ops = { 6748 .start = bkn_seq_dma_start, 6749 .next = bkn_seq_dma_next, 6750 .stop = bkn_seq_dma_stop, 6751 .show = bkn_seq_dma_show 6752 }; 6753 6754 static int 6755 bkn_seq_dma_open(struct inode *inode, struct file *file) 6756 { 6757 return seq_open(file, &bkn_seq_dma_ops); 6758 }; 6759 6760 static struct file_operations bkn_seq_dma_file_ops = { 6761 .owner = THIS_MODULE, 6762 .open = bkn_seq_dma_open, 6763 .read = seq_read, 6764 .llseek = seq_lseek, 6765 .release = seq_release 6766 }; 6767 6768 /* 6769 * Device Debug Control Proc Write Entry 6770 * 6771 * Syntax: 6772 * [<unit>:]debug=<mask> 6773 * 6774 * Where <mask> corresponds to the debug module parameter. 6775 * 6776 * Examples: 6777 * debug=0xffff 6778 * 0:debug-0x2000 6779 */ 6780 static ssize_t 6781 bkn_proc_debug_write(struct file *file, const char *buf, 6782 size_t count, loff_t *loff) 6783 { 6784 bkn_switch_info_t *sinfo; 6785 char debug_str[40]; 6786 char *ptr; 6787 int unit; 6788 6789 if (count > sizeof(debug_str)) { 6790 count = sizeof(debug_str) - 1; 6791 debug_str[count] = '\0'; 6792 } 6793 if (copy_from_user(debug_str, buf, count)) { 6794 return -EFAULT; 6795 } 6796 6797 unit = simple_strtol(debug_str, NULL, 10); 6798 sinfo = bkn_sinfo_from_unit(unit); 6799 if (sinfo == NULL) { 6800 gprintk("Warning: unknown unit: %d\n", unit); 6801 return count; 6802 } 6803 6804 if ((ptr = strstr(debug_str, "debug=")) != NULL) { 6805 ptr += 6; 6806 debug = simple_strtol(ptr, NULL, 0); 6807 } else { 6808 gprintk("Warning: unknown configuration setting\n"); 6809 } 6810 6811 return count; 6812 } 6813 6814 /* 6815 * Driver Debug Proc Entry 6816 */ 6817 static int 6818 bkn_proc_debug_show(struct seq_file *m, void *v) 6819 { 6820 int unit = 0; 6821 struct list_head *list; 6822 bkn_switch_info_t *sinfo; 6823 6824 seq_printf(m, "Configuration:\n"); 6825 seq_printf(m, " debug: 0x%x\n", debug); 6826 seq_printf(m, " mac_addr: %02x:%02x:%02x:%02x:%02x:%02x\n", 6827 bkn_dev_mac[0], bkn_dev_mac[1], bkn_dev_mac[2], 6828 bkn_dev_mac[3], bkn_dev_mac[4], bkn_dev_mac[5]); 6829 seq_printf(m, " rx_buffer_size: %d (0x%x)\n", 6830 rx_buffer_size, rx_buffer_size); 6831 seq_printf(m, " rcpu_mode: %d\n", rcpu_mode); 6832 seq_printf(m, " rcpu_dmac: %02x:%02x:%02x:%02x:%02x:%02x\n", 6833 bkn_rcpu_dmac[0], bkn_rcpu_dmac[1], bkn_rcpu_dmac[2], 6834 bkn_rcpu_dmac[3], bkn_rcpu_dmac[4], bkn_rcpu_dmac[5]); 6835 seq_printf(m, " rcpu_smac: %02x:%02x:%02x:%02x:%02x:%02x\n", 6836 bkn_rcpu_smac[0], bkn_rcpu_smac[1], bkn_rcpu_smac[2], 6837 bkn_rcpu_smac[3], bkn_rcpu_smac[4], bkn_rcpu_smac[5]); 6838 seq_printf(m, " rcpu_ethertype: 0x%x\n", rcpu_ethertype); 6839 seq_printf(m, " rcpu_signature: 0x%x\n", rcpu_signature); 6840 seq_printf(m, " rcpu_vlan: %d\n", rcpu_vlan); 6841 seq_printf(m, " use_rx_skb: %d\n", use_rx_skb); 6842 seq_printf(m, " num_rx_prio: %d\n", num_rx_prio); 6843 seq_printf(m, " check_rcpu_sig: %d\n", check_rcpu_signature); 6844 seq_printf(m, " default_mtu: %d\n", default_mtu); 6845 seq_printf(m, " rx_sync_retry: %d\n", rx_sync_retry); 6846 seq_printf(m, " use_napi: %d\n", use_napi); 6847 seq_printf(m, " napi_weight: %d\n", napi_weight); 6848 seq_printf(m, " basedev_susp: %d\n", basedev_suspend); 6849 seq_printf(m, " force_tagged: %d\n", force_tagged); 6850 seq_printf(m, " ft_tpid: %d\n", ft_tpid); 6851 seq_printf(m, " ft_pri: %d\n", ft_pri); 6852 seq_printf(m, " ft_pri: %d\n", ft_cfi); 6853 seq_printf(m, " ft_tpid: %d\n", ft_vid); 6854 seq_printf(m, "Active IOCTLs:\n"); 6855 seq_printf(m, " Command: %d\n", ioctl_cmd); 6856 seq_printf(m, " Event: %d\n", ioctl_evt); 6857 6858 list_for_each(list, &_sinfo_list) { 6859 sinfo = (bkn_switch_info_t *)list; 6860 6861 seq_printf(m, "Device %d:\n", unit); 6862 seq_printf(m, " base_addr: 0x%p\n", sinfo->base_addr); 6863 seq_printf(m, " dev_no: %d\n", sinfo->dev_no); 6864 seq_printf(m, " cmic_type: %c\n", sinfo->cmic_type); 6865 seq_printf(m, " dcb_type: %d\n", sinfo->dcb_type); 6866 seq_printf(m, " dcb_wsize: %d\n", sinfo->dcb_wsize); 6867 seq_printf(m, " pkt_hdr_size: %d\n", sinfo->pkt_hdr_size); 6868 seq_printf(m, " rx_chans: %d\n", sinfo->rx_chans); 6869 seq_printf(m, " cdma_chans: 0x%x\n", sinfo->cdma_channels); 6870 seq_printf(m, " irq_mask: 0x%x\n", sinfo->irq_mask); 6871 seq_printf(m, " dma_events: 0x%x\n", sinfo->dma_events); 6872 seq_printf(m, " dcb_dma: 0x%p\n", (void *)(sal_paddr_t)sinfo->dcb_dma); 6873 seq_printf(m, " dcb_mem_size: 0x%x\n", sinfo->dcb_mem_size); 6874 seq_printf(m, " rcpu_sig: 0x%x\n", sinfo->rcpu_sig); 6875 seq_printf(m, " napi_poll_mode: %d\n", sinfo->napi_poll_mode); 6876 seq_printf(m, " inst_id: 0x%x\n", sinfo->inst_id); 6877 seq_printf(m, " evt_queue: %d\n", sinfo->evt_idx); 6878 6879 unit++; 6880 } 6881 6882 return 0; 6883 } 6884 6885 static int bkn_proc_debug_open(struct inode * inode, struct file * file) 6886 { 6887 return single_open(file, bkn_proc_debug_show, NULL); 6888 } 6889 6890 struct file_operations bkn_proc_debug_file_ops = { 6891 owner: THIS_MODULE, 6892 open: bkn_proc_debug_open, 6893 read: seq_read, 6894 llseek: seq_lseek, 6895 write: bkn_proc_debug_write, 6896 release: single_release, 6897 }; 6898 6899 /* 6900 * Device Statistics Proc Entry 6901 */ 6902 static int 6903 bkn_proc_stats_show(struct seq_file *m, void *v) 6904 { 6905 int unit = 0; 6906 struct list_head *list, *flist; 6907 bkn_switch_info_t *sinfo; 6908 bkn_filter_t *filter; 6909 int chan; 6910 6911 6912 list_for_each(list, &_sinfo_list) { 6913 sinfo = (bkn_switch_info_t *)list; 6914 6915 seq_printf(m, "Device stats (unit %d):\n", unit); 6916 seq_printf(m, " Interrupts %10u\n", sinfo->interrupts); 6917 seq_printf(m, " Tx packets %10u\n", sinfo->tx.pkts); 6918 for (chan = 0; chan < sinfo->rx_chans; chan++) { 6919 seq_printf(m, " Rx%d packets %10u\n", chan, sinfo->rx[chan].pkts); 6920 } 6921 for (chan = 0; chan < sinfo->rx_chans; chan++) { 6922 if (sinfo->interrupts == 0) { 6923 /* Avoid divide-by-zero */ 6924 seq_printf(m, " Rx%d pkts/intr -\n", chan); 6925 } else { 6926 seq_printf(m, " Rx%d pkts/intr %8u\n", 6927 chan, sinfo->rx[chan].pkts / sinfo->interrupts); 6928 } 6929 } 6930 seq_printf(m, " Timer runs %10u\n", sinfo->timer_runs); 6931 seq_printf(m, " NAPI reruns %10u\n", sinfo->napi_not_done); 6932 6933 list_for_each(flist, &sinfo->rxpf_list) { 6934 filter = (bkn_filter_t *)flist; 6935 6936 seq_printf(m, " Filter %d stats:\n", filter->kf.id); 6937 seq_printf(m, " Hits %10lu\n", filter->hits); 6938 } 6939 6940 unit++; 6941 } 6942 return 0; 6943 } 6944 6945 static int bkn_proc_stats_open(struct inode * inode, struct file * file) 6946 { 6947 return single_open(file, bkn_proc_stats_show, NULL); 6948 } 6949 6950 /* 6951 * Device Statistics Proc Write Entry 6952 * 6953 * Syntax: 6954 * [<unit>:]clear[=all] 6955 * 6956 * Where <mask> corresponds to the debug module parameter. 6957 * 6958 * Examples: 6959 * clear 6960 * 0:clear=all 6961 */ 6962 static ssize_t 6963 bkn_proc_stats_write(struct file *file, const char *buf, 6964 size_t count, loff_t *loff) 6965 { 6966 bkn_switch_info_t *sinfo; 6967 struct list_head *flist; 6968 bkn_filter_t *filter; 6969 char debug_str[40]; 6970 char *ptr; 6971 int unit; 6972 int clear_mask; 6973 int chan; 6974 6975 if (count > sizeof(debug_str)) { 6976 count = sizeof(debug_str) - 1; 6977 debug_str[count] = '\0'; 6978 } 6979 if (copy_from_user(debug_str, buf, count)) { 6980 return -EFAULT; 6981 } 6982 6983 unit = simple_strtol(debug_str, NULL, 10); 6984 sinfo = bkn_sinfo_from_unit(unit); 6985 if (sinfo == NULL) { 6986 gprintk("Warning: unknown unit: %d\n", unit); 6987 return count; 6988 } 6989 6990 clear_mask = 0; 6991 if ((ptr = strstr(debug_str, "clear=")) != NULL) { 6992 ptr += 6; 6993 if (strncmp(ptr, "all", 3) == 0) { 6994 clear_mask = ~0; 6995 } 6996 } else if ((ptr = strstr(debug_str, "clear")) != NULL) { 6997 clear_mask = ~0; 6998 } else { 6999 gprintk("Warning: unknown configuration setting\n"); 7000 } 7001 7002 if (clear_mask) { 7003 sinfo->tx.pkts = 0; 7004 for (chan = 0; chan < sinfo->rx_chans; chan++) { 7005 sinfo->rx[chan].pkts = 0; 7006 } 7007 sinfo->interrupts = 0; 7008 sinfo->timer_runs = 0; 7009 sinfo->napi_not_done = 0; 7010 list_for_each(flist, &sinfo->rxpf_list) { 7011 filter = (bkn_filter_t *)flist; 7012 filter->hits = 0; 7013 } 7014 } 7015 7016 return count; 7017 } 7018 7019 struct file_operations bkn_proc_stats_file_ops = { 7020 owner: THIS_MODULE, 7021 open: bkn_proc_stats_open, 7022 read: seq_read, 7023 llseek: seq_lseek, 7024 write: bkn_proc_stats_write, 7025 release: single_release, 7026 }; 7027 7028 /* 7029 * Device Debug Statistics Proc Entry 7030 */ 7031 static int 7032 bkn_proc_dstats_show(struct seq_file *m, void *v) 7033 { 7034 int unit = 0; 7035 struct list_head *list; 7036 bkn_switch_info_t *sinfo; 7037 int chan; 7038 7039 list_for_each(list, &_sinfo_list) { 7040 sinfo = (bkn_switch_info_t *)list; 7041 7042 seq_printf(m, "Device debug stats (unit %d):\n", unit); 7043 seq_printf(m, " Tx drop no skb %10u\n", 7044 sinfo->tx.pkts_d_no_skb); 7045 seq_printf(m, " Tx drop rcpu encap %10u\n", 7046 sinfo->tx.pkts_d_rcpu_encap); 7047 seq_printf(m, " Tx drop rcpu sig %10u\n", 7048 sinfo->tx.pkts_d_rcpu_sig); 7049 seq_printf(m, " Tx drop rcpu meta %10u\n", 7050 sinfo->tx.pkts_d_rcpu_meta); 7051 seq_printf(m, " Tx drop pad failed %10u\n", 7052 sinfo->tx.pkts_d_pad_fail); 7053 seq_printf(m, " Tx drop no resource %10u\n", 7054 sinfo->tx.pkts_d_dma_resrc); 7055 seq_printf(m, " Tx drop callback %10u\n", 7056 sinfo->tx.pkts_d_callback); 7057 seq_printf(m, " Tx drop no link %10u\n", 7058 sinfo->tx.pkts_d_no_link); 7059 seq_printf(m, " Tx drop oversized %10u\n", 7060 sinfo->tx.pkts_d_over_limit); 7061 seq_printf(m, " Tx suspends %10u\n", 7062 sinfo->tx.suspends); 7063 for (chan = 0; chan < sinfo->rx_chans; chan++) { 7064 seq_printf(m, " Rx%d filter to api %10u\n", 7065 chan, sinfo->rx[chan].pkts_f_api); 7066 seq_printf(m, " Rx%d filter to netif %10u\n", 7067 chan, sinfo->rx[chan].pkts_f_netif); 7068 seq_printf(m, " Rx%d mirror to api %10u\n", 7069 chan, sinfo->rx[chan].pkts_m_api); 7070 seq_printf(m, " Rx%d mirror to netif %10u\n", 7071 chan, sinfo->rx[chan].pkts_m_netif); 7072 seq_printf(m, " Rx%d drop no skb %10u\n", 7073 chan, sinfo->rx[chan].pkts_d_no_skb); 7074 seq_printf(m, " Rx%d drop no match %10u\n", 7075 chan, sinfo->rx[chan].pkts_d_no_match); 7076 seq_printf(m, " Rx%d drop unkn netif %10u\n", 7077 chan, sinfo->rx[chan].pkts_d_unkn_netif); 7078 seq_printf(m, " Rx%d drop unkn dest %10u\n", 7079 chan, sinfo->rx[chan].pkts_d_unkn_dest); 7080 seq_printf(m, " Rx%d drop callback %10u\n", 7081 chan, sinfo->rx[chan].pkts_d_callback); 7082 seq_printf(m, " Rx%d drop no link %10u\n", 7083 chan, sinfo->rx[chan].pkts_d_no_link); 7084 seq_printf(m, " Rx%d sync error %10u\n", 7085 chan, sinfo->rx[chan].sync_err); 7086 seq_printf(m, " Rx%d sync retry %10u\n", 7087 chan, sinfo->rx[chan].sync_retry); 7088 seq_printf(m, " Rx%d sync maxloop %10u\n", 7089 chan, sinfo->rx[chan].sync_maxloop); 7090 seq_printf(m, " Rx%d drop no buffer %10u\n", 7091 chan, sinfo->rx[chan].pkts_d_no_api_buf); 7092 } 7093 unit++; 7094 } 7095 return 0; 7096 } 7097 7098 static int bkn_proc_dstats_open(struct inode * inode, struct file * file) 7099 { 7100 return single_open(file, bkn_proc_dstats_show, NULL); 7101 } 7102 7103 /* 7104 * Device Debug Statistics Proc Write Entry 7105 * 7106 * Syntax: 7107 * [<unit>:]clear[=all|tx|rx[<chan>]] 7108 * 7109 * Where <mask> corresponds to the debug module parameter. 7110 * 7111 * Examples: 7112 * clear 7113 * 0:clear=rx1 7114 */ 7115 static ssize_t 7116 bkn_proc_dstats_write(struct file *file, const char *buf, 7117 size_t count, loff_t *loff) 7118 { 7119 bkn_switch_info_t *sinfo; 7120 char debug_str[40]; 7121 char *ptr; 7122 int unit; 7123 int clear_mask; 7124 int chan; 7125 7126 if (count > sizeof(debug_str)) { 7127 count = sizeof(debug_str) - 1; 7128 debug_str[count] = '\0'; 7129 } 7130 if (copy_from_user(debug_str, buf, count)) { 7131 return -EFAULT; 7132 } 7133 7134 unit = simple_strtol(debug_str, NULL, 10); 7135 sinfo = bkn_sinfo_from_unit(unit); 7136 if (sinfo == NULL) { 7137 gprintk("Warning: unknown unit: %d\n", unit); 7138 return count; 7139 } 7140 7141 clear_mask = 0; 7142 if ((ptr = strstr(debug_str, "clear=")) != NULL) { 7143 ptr += 6; 7144 if (strncmp(ptr, "all", 3) == 0) { 7145 clear_mask = ~0; 7146 } else if (strncmp(ptr, "dev", 3) == 0) { 7147 clear_mask = 0x20; 7148 } else if (strncmp(ptr, "tx", 2) == 0) { 7149 clear_mask = 0x10; 7150 } else if (strncmp(ptr, "rx0", 3) == 0) { 7151 clear_mask = (1 << 0); 7152 } else if (strncmp(ptr, "rx1", 3) == 0) { 7153 clear_mask = (1 << 1); 7154 } else if (strncmp(ptr, "rx", 2) == 0) { 7155 clear_mask = 0xf; 7156 } 7157 } else if ((ptr = strstr(debug_str, "clear")) != NULL) { 7158 clear_mask = ~0; 7159 } else { 7160 gprintk("Warning: unknown configuration setting\n"); 7161 } 7162 7163 /* Tx counters */ 7164 if (clear_mask & 0x10) { 7165 sinfo->tx.pkts_d_no_skb = 0; 7166 sinfo->tx.pkts_d_rcpu_encap = 0; 7167 sinfo->tx.pkts_d_rcpu_sig = 0; 7168 sinfo->tx.pkts_d_rcpu_meta = 0; 7169 sinfo->tx.pkts_d_pad_fail = 0; 7170 sinfo->tx.pkts_d_over_limit = 0; 7171 sinfo->tx.pkts_d_dma_resrc = 0; 7172 sinfo->tx.suspends = 0; 7173 } 7174 /* Rx counters */ 7175 for (chan = 0; chan < sinfo->rx_chans; chan++) { 7176 if (clear_mask & (1 << chan)) { 7177 sinfo->rx[chan].pkts_f_api = 0; 7178 sinfo->rx[chan].pkts_f_netif = 0; 7179 sinfo->rx[chan].pkts_m_api = 0; 7180 sinfo->rx[chan].pkts_m_netif = 0; 7181 sinfo->rx[chan].pkts_d_no_skb = 0; 7182 sinfo->rx[chan].pkts_d_no_match = 0; 7183 sinfo->rx[chan].pkts_d_unkn_netif = 0; 7184 sinfo->rx[chan].pkts_d_unkn_dest = 0; 7185 sinfo->rx[chan].pkts_d_no_api_buf = 0; 7186 sinfo->rx[chan].sync_err = 0; 7187 sinfo->rx[chan].sync_retry = 0; 7188 sinfo->rx[chan].sync_maxloop = 0; 7189 } 7190 } 7191 7192 return count; 7193 } 7194 7195 struct file_operations bkn_proc_dstats_file_ops = { 7196 owner: THIS_MODULE, 7197 open: bkn_proc_dstats_open, 7198 read: seq_read, 7199 llseek: seq_lseek, 7200 write: bkn_proc_dstats_write, 7201 release: single_release, 7202 }; 7203 7204 static int 7205 bkn_proc_init(void) 7206 { 7207 struct proc_dir_entry *entry; 7208 7209 PROC_CREATE(entry, "link", 0666, bkn_proc_root, &bkn_proc_link_file_ops); 7210 if (entry == NULL) { 7211 return -1; 7212 } 7213 PROC_CREATE(entry, "rate", 0666, bkn_proc_root, &bkn_proc_rate_file_ops); 7214 if (entry == NULL) { 7215 return -1; 7216 } 7217 PROC_CREATE(entry, "dma", 0, bkn_proc_root, &bkn_seq_dma_file_ops); 7218 if (entry == NULL) { 7219 return -1; 7220 } 7221 PROC_CREATE(entry, "debug", 0666, bkn_proc_root, &bkn_proc_debug_file_ops); 7222 if (entry == NULL) { 7223 return -1; 7224 } 7225 PROC_CREATE(entry, "stats", 0666, bkn_proc_root, &bkn_proc_stats_file_ops); 7226 if (entry == NULL) { 7227 return -1; 7228 } 7229 PROC_CREATE(entry, "dstats", 0666, bkn_proc_root, &bkn_proc_dstats_file_ops); 7230 if (entry == NULL) { 7231 return -1; 7232 } 7233 7234 return 0; 7235 } 7236 7237 static int 7238 bkn_proc_cleanup(void) 7239 { 7240 remove_proc_entry("link", bkn_proc_root); 7241 remove_proc_entry("rate", bkn_proc_root); 7242 remove_proc_entry("dma", bkn_proc_root); 7243 remove_proc_entry("debug", bkn_proc_root); 7244 remove_proc_entry("stats", bkn_proc_root); 7245 remove_proc_entry("dstats", bkn_proc_root); 7246 return 0; 7247 } 7248 7249 /* 7250 * Generic module functions 7251 */ 7252 7253 static int 7254 _pprint(void) 7255 { 7256 pprintf("Broadcom BCM KNET Linux Network Driver\n"); 7257 7258 return 0; 7259 } 7260 7261 static int 7262 bkn_knet_dma_info(kcom_msg_dma_info_t *kmsg, int len) 7263 { 7264 bkn_switch_info_t *sinfo; 7265 bkn_dcb_chain_t *dcb_chain, *dcb_chain_end; 7266 unsigned long flags; 7267 int chan; 7268 uint64_t dcb_dma; 7269 int woffset; 7270 7271 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7272 7273 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7274 if (sinfo == NULL) { 7275 kmsg->hdr.status = KCOM_E_PARAM; 7276 return sizeof(kcom_msg_hdr_t); 7277 } 7278 7279 dcb_chain = kmalloc(sizeof(*dcb_chain), GFP_KERNEL); 7280 if (dcb_chain == NULL) { 7281 gprintk("Fatal error: No memory for dcb_chain\n"); 7282 kmsg->hdr.status = KCOM_E_RESOURCE; 7283 return sizeof(kcom_msg_hdr_t); 7284 } 7285 memset(dcb_chain, 0, sizeof(*dcb_chain)); 7286 dcb_chain->dcb_cnt = kmsg->dma_info.cnt; 7287 dcb_chain->dcb_dma = kmsg->dma_info.data.dcb_start; 7288 dcb_chain->dcb_mem = kernel_bde->p2l(sinfo->dev_no, 7289 (sal_paddr_t)dcb_chain->dcb_dma); 7290 7291 if (kmsg->dma_info.type == KCOM_DMA_INFO_T_TX_DCB) { 7292 spin_lock_irqsave(&sinfo->lock, flags); 7293 7294 /* Hold back packets from kernel */ 7295 bkn_suspend_tx(sinfo); 7296 7297 list_add_tail(&dcb_chain->list, &sinfo->tx.api_dcb_list); 7298 7299 /* Handle for Continuous DMA mode */ 7300 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN)) { 7301 woffset = (dcb_chain->dcb_cnt - 1) * sinfo->dcb_wsize + 1; 7302 if (sinfo->cmic_type == 'x') { 7303 woffset += 1; 7304 } 7305 if ((dcb_chain->dcb_mem[woffset] & ((1 << 18) | (1 << 16))) != 0x50000) { 7306 gprintk("Not suitable API DCB chain for Continuous DMA mode\n"); 7307 list_del(&dcb_chain->list); 7308 kfree(dcb_chain); 7309 kmsg->hdr.status = KCOM_E_PARAM; 7310 spin_unlock_irqrestore(&sinfo->lock, flags); 7311 return sizeof(kcom_msg_hdr_t); 7312 } 7313 dcb_chain_end = sinfo->tx.api_dcb_chain_end; 7314 if (dcb_chain_end != NULL) { 7315 /* Stitch this chain */ 7316 woffset = (dcb_chain_end->dcb_cnt - 1) * sinfo->dcb_wsize; 7317 dcb_chain_end->dcb_mem[woffset] = dcb_chain->dcb_dma; 7318 if (sinfo->cmic_type == 'x') { 7319 dcb_chain_end->dcb_mem[woffset + 1] = DMA_TO_BUS_HI(dcb_chain->dcb_dma >> 32); 7320 } 7321 MEMORY_BARRIER; 7322 } 7323 sinfo->tx.api_dcb_chain_end = dcb_chain; 7324 if (sinfo->tx.api_active) { 7325 /* Set new halt location */ 7326 woffset = (dcb_chain->dcb_cnt - 1) * sinfo->dcb_wsize; 7327 dcb_dma = dcb_chain->dcb_dma + woffset * sizeof(uint32_t); 7328 /* DMA run to the new halt location */ 7329 bkn_cdma_goto(sinfo, XGS_DMA_TX_CHAN, dcb_dma); 7330 } 7331 } 7332 7333 if (sinfo->tx.free == MAX_TX_DCBS && 7334 !sinfo->tx.api_active && 7335 !sinfo->basedev_suspended) { 7336 bkn_api_tx(sinfo); 7337 } 7338 7339 spin_unlock_irqrestore(&sinfo->lock, flags); 7340 } else if (kmsg->dma_info.type == KCOM_DMA_INFO_T_RX_DCB) { 7341 spin_lock_irqsave(&sinfo->lock, flags); 7342 7343 chan = kmsg->dma_info.chan - 1; 7344 if ((chan < 0) || (chan > sinfo->rx_chans)) { 7345 gprintk("Invalid RX DMA channel specified: %d\n", 7346 kmsg->dma_info.chan); 7347 kmsg->hdr.status = KCOM_E_PARAM; 7348 spin_unlock_irqrestore(&sinfo->lock, flags); 7349 return sizeof(kcom_msg_hdr_t); 7350 } 7351 7352 list_add_tail(&dcb_chain->list, &sinfo->rx[chan].api_dcb_list); 7353 7354 /* Handle for Continuous DMA mode */ 7355 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 7356 woffset = (dcb_chain->dcb_cnt - 1) * sinfo->dcb_wsize + 1; 7357 if (sinfo->cmic_type == 'x') { 7358 woffset += 1; 7359 } 7360 if ((dcb_chain->dcb_mem[woffset] & ((1 << 18) | (1 << 16))) != 0x50000) { 7361 gprintk("Not suitable API DCB chain for Continuous DMA mode\n"); 7362 list_del(&dcb_chain->list); 7363 kfree(dcb_chain); 7364 kmsg->hdr.status = KCOM_E_PARAM; 7365 spin_unlock_irqrestore(&sinfo->lock, flags); 7366 return sizeof(kcom_msg_hdr_t); 7367 } 7368 dcb_chain_end = sinfo->rx[chan].api_dcb_chain_end; 7369 if (dcb_chain_end != NULL) { 7370 /* Stitch this chain */ 7371 woffset = (dcb_chain_end->dcb_cnt - 1) * sinfo->dcb_wsize; 7372 dcb_chain_end->dcb_mem[woffset] = dcb_chain->dcb_dma; 7373 if (sinfo->cmic_type == 'x') { 7374 dcb_chain_end->dcb_mem[woffset + 1] = DMA_TO_BUS_HI(dcb_chain->dcb_dma >> 32); 7375 } 7376 MEMORY_BARRIER; 7377 } 7378 sinfo->rx[chan].api_dcb_chain_end = dcb_chain; 7379 if (!sinfo->rx[chan].use_rx_skb) { 7380 /* Set new halt location */ 7381 woffset = (dcb_chain->dcb_cnt - 1) * sinfo->dcb_wsize; 7382 dcb_dma = dcb_chain->dcb_dma + woffset * sizeof(uint32_t); 7383 /* DMA run to the new halt location */ 7384 bkn_cdma_goto(sinfo, XGS_DMA_RX_CHAN + chan, dcb_dma); 7385 } 7386 } 7387 7388 if (sinfo->rx[chan].api_active == 0) { 7389 bkn_api_rx_restart(sinfo, chan); 7390 } 7391 7392 spin_unlock_irqrestore(&sinfo->lock, flags); 7393 } else { 7394 DBG_DCB(("Unknown DCB_INFO type (%d).\n", kmsg->dma_info.type)); 7395 kfree(dcb_chain); 7396 kmsg->hdr.status = KCOM_E_PARAM; 7397 } 7398 7399 return sizeof(kcom_msg_hdr_t); 7400 } 7401 7402 static int 7403 bkn_create_inst(uint32 inst_id) 7404 { 7405 bkn_switch_info_t *sinfo; 7406 bkn_evt_resource_t *evt; 7407 unsigned long flags; 7408 int i, evt_idx = -1; 7409 7410 /* multiple instance mode */ 7411 for (i = 0; i < kernel_bde->num_devices(BDE_ALL_DEVICES); i++) { 7412 evt = &_bkn_evt[i]; 7413 if (evt->inst_id == inst_id) { 7414 evt_idx = i; 7415 DBG_INST(("%s evt_idx %d inst_id 0x%x\n",__FUNCTION__, i, inst_id)); 7416 break; 7417 } 7418 if ((_bkn_multi_inst == 0) || (evt->inst_id == 0)) { 7419 _bkn_multi_inst ++; 7420 evt_idx = i; 7421 init_waitqueue_head(&evt->evt_wq); 7422 evt->inst_id = inst_id; 7423 DBG_INST(("%s evt_idx %d inst_id 0x%x\n",__FUNCTION__, i, inst_id)); 7424 break; 7425 } 7426 } 7427 7428 if (evt_idx == -1) { 7429 DBG_WARN(("Run out the event queue resource !\n")); 7430 return -1; 7431 } 7432 for (i = 0; i < kernel_bde->num_devices(BDE_ALL_DEVICES); i++) { 7433 if (inst_id & (1 << i)) { 7434 sinfo = bkn_sinfo_from_unit(i); 7435 spin_lock_irqsave(&sinfo->lock, flags); 7436 sinfo->evt_idx = evt_idx; 7437 spin_unlock_irqrestore(&sinfo->lock, flags); 7438 DBG_INST(("%s d(%d) evt_idx %d \n",__FUNCTION__, i, evt_idx)); 7439 } 7440 } 7441 return 0; 7442 } 7443 7444 /* 7445 * Device reprobe driven by application to check if new device is probed or 7446 * existed device is changed after inserting KNET module. 7447 */ 7448 static int 7449 bkn_knet_dev_reprobe(void) 7450 { 7451 bkn_switch_info_t *sinfo; 7452 int i; 7453 7454 for (i = 0; i < kernel_bde->num_devices(BDE_ALL_DEVICES); i++) { 7455 sinfo = bkn_sinfo_from_unit(i); 7456 if (sinfo == NULL) { 7457 /* New device found after re-probe. */ 7458 if (bkn_knet_dev_init(i) < 0) { 7459 return -1; 7460 } 7461 } else { 7462 /* Existed device reinit after re-probe. */ 7463 if (bkn_knet_dev_reinit(i) < 0) { 7464 return -1; 7465 } 7466 } 7467 } 7468 return 0; 7469 } 7470 7471 /* Assign the inst_id and evt_idx */ 7472 static int 7473 bkn_knet_dev_inst_set(kcom_msg_reprobe_t *kmsg) 7474 { 7475 bkn_switch_info_t *sinfo; 7476 int d = kmsg->hdr.unit; 7477 uint32 inst = 0; 7478 unsigned long flags; 7479 struct list_head *list; 7480 7481 sinfo = bkn_sinfo_from_unit(d); 7482 lkbde_dev_instid_get(d, &inst); 7483 7484 spin_lock_irqsave(&sinfo->lock, flags); 7485 if (sinfo->inst_id != inst) { 7486 /* Instance database changed, reinit the inst_id */ 7487 sinfo->inst_id = 0; 7488 sinfo->evt_idx = -1; 7489 } 7490 spin_unlock_irqrestore(&sinfo->lock, flags); 7491 7492 if (inst) { 7493 if (sinfo->inst_id == 0) { 7494 if (bkn_create_inst(inst) != 0) { 7495 return -1; 7496 } 7497 } 7498 spin_lock_irqsave(&sinfo->lock, flags); 7499 sinfo->inst_id = inst; 7500 spin_unlock_irqrestore(&sinfo->lock, flags); 7501 } else { 7502 /* legacy mode */ 7503 list_for_each(list, &_sinfo_list) { 7504 sinfo = (bkn_switch_info_t *)list; 7505 spin_lock_irqsave(&sinfo->lock, flags); 7506 sinfo->evt_idx = 0; 7507 sinfo->inst_id = 0; 7508 spin_unlock_irqrestore(&sinfo->lock, flags); 7509 } 7510 } 7511 return 0; 7512 } 7513 7514 static int 7515 bkn_knet_version(kcom_msg_version_t *kmsg, int len) 7516 { 7517 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7518 kmsg->version = KCOM_VERSION; 7519 kmsg->netif_max = KCOM_NETIF_MAX; 7520 kmsg->filter_max = KCOM_FILTER_MAX; 7521 7522 return sizeof(kcom_msg_version_t); 7523 } 7524 7525 static int 7526 bkn_knet_hw_reset(kcom_msg_hw_reset_t *kmsg, int len) 7527 { 7528 bkn_switch_info_t *sinfo; 7529 unsigned long flags; 7530 int chan; 7531 7532 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7533 7534 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7535 if (sinfo == NULL) { 7536 kmsg->hdr.status = KCOM_E_PARAM; 7537 return sizeof(kcom_msg_hdr_t); 7538 } 7539 7540 spin_lock_irqsave(&sinfo->lock, flags); 7541 7542 if (kmsg->channels == 0) { 7543 /* Clean all if no channels specified */ 7544 bkn_dma_abort(sinfo); 7545 bkn_clean_dcbs(sinfo); 7546 skb_queue_purge(&sinfo->tx_ptp_queue); 7547 } else { 7548 if (kmsg->channels & (1 << XGS_DMA_TX_CHAN)) { 7549 bkn_dma_abort_tx(sinfo); 7550 bkn_clean_tx_dcbs(sinfo); 7551 skb_queue_purge(&sinfo->tx_ptp_queue); 7552 } 7553 for (chan = 0; chan < sinfo->rx_chans; chan++) { 7554 if (kmsg->channels & (1 << (XGS_DMA_RX_CHAN + chan))) { 7555 bkn_dma_abort_rx(sinfo, chan); 7556 bkn_clean_rx_dcbs(sinfo, chan); 7557 } 7558 } 7559 } 7560 7561 spin_unlock_irqrestore(&sinfo->lock, flags); 7562 7563 return sizeof(kcom_msg_hdr_t); 7564 } 7565 7566 static int 7567 bkn_knet_hw_init(kcom_msg_hw_init_t *kmsg, int len) 7568 { 7569 bkn_switch_info_t *sinfo; 7570 uint32_t dev_type; 7571 unsigned long flags; 7572 7573 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7574 7575 DBG_DCB(("DCB size %d, type %d\n", kmsg->dcb_size, kmsg->dcb_type)); 7576 7577 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7578 if (sinfo == NULL) { 7579 kmsg->hdr.status = KCOM_E_PARAM; 7580 return sizeof(kcom_msg_hdr_t); 7581 } 7582 7583 spin_lock_irqsave(&sinfo->lock, flags); 7584 7585 sinfo->cmic_type = kmsg->cmic_type; 7586 sinfo->dcb_type = kmsg->dcb_type; 7587 sinfo->dcb_wsize = BYTES2WORDS(kmsg->dcb_size); 7588 sinfo->pkt_hdr_size = kmsg->pkt_hdr_size; 7589 sinfo->dma_hi = kmsg->dma_hi; 7590 sinfo->rx_chans = sinfo->cmic_type == 'x' ? NUM_CMICX_RX_CHAN : NUM_CMICM_RX_CHAN; 7591 if (sinfo->rx_chans > NUM_RX_CHAN) { 7592 sinfo->rx_chans = NUM_RX_CHAN; 7593 } 7594 7595 DBG_DUNE(("CMIC:%c DCB:%d WSIZE:%d DMA HI: 0x%08x HDR size: %d\n", 7596 sinfo->cmic_type, sinfo->dcb_type, sinfo->dcb_wsize, 7597 sinfo->dma_hi, sinfo->pkt_hdr_size)); 7598 7599 /* Config Continuous DMA mode */ 7600 sinfo->cdma_channels = kmsg->cdma_channels & ~(~0 << (sinfo->rx_chans + 1)); 7601 7602 /* Ensure 32-bit PCI DMA is mapped properly on 64-bit platforms */ 7603 dev_type = kernel_bde->get_dev_type(sinfo->dev_no); 7604 if (dev_type & BDE_PCI_DEV_TYPE && sinfo->cmic_type != 'x') { 7605 if (pci_set_dma_mask(sinfo->pdev, 0xffffffff)) { 7606 spin_unlock_irqrestore(&sinfo->lock, flags); 7607 gprintk("No suitable DMA available for SKBs\n"); 7608 kmsg->hdr.status = KCOM_E_RESOURCE; 7609 return sizeof(kcom_msg_hdr_t); 7610 } 7611 } 7612 7613 /* First time called we need to allocate DCBs */ 7614 if (sinfo->dcb_mem == NULL) { 7615 if (bkn_alloc_dcbs(sinfo) < 0) { 7616 spin_unlock_irqrestore(&sinfo->lock, flags); 7617 kmsg->hdr.status = KCOM_E_RESOURCE; 7618 return sizeof(kcom_msg_hdr_t); 7619 } 7620 } 7621 7622 if (device_is_sand(sinfo)) { 7623 int idx = 0; 7624 /* Information to parser Dune system headers */ 7625 sinfo->ftmh_lb_key_ext_size = kmsg->ftmh_lb_key_ext_size; 7626 sinfo->ftmh_stacking_ext_size = kmsg->ftmh_stacking_ext_size; 7627 sinfo->pph_base_size = kmsg->pph_base_size; 7628 for (idx = 0; idx < 8; idx++) 7629 { 7630 sinfo->pph_lif_ext_size[idx] = kmsg->pph_lif_ext_size[idx]; 7631 } 7632 for (idx = 0; idx < 4; idx++) 7633 { 7634 sinfo->udh_length_type[idx] = kmsg->udh_length_type[idx]; 7635 } 7636 sinfo->udh_size = kmsg->udh_size; 7637 sinfo->oamp_punt = kmsg->oamp_punted; 7638 sinfo->no_skip_udh_check = kmsg->no_skip_udh_check; 7639 sinfo->system_headers_mode = kmsg->system_headers_mode; 7640 sinfo->udh_enable = kmsg->udh_enable; 7641 } 7642 7643 /* Ensure that we restart properly */ 7644 bkn_dma_abort(sinfo); 7645 bkn_clean_dcbs(sinfo); 7646 7647 if (basedev_suspend) { 7648 if (!netif_running(sinfo->dev)) { 7649 sinfo->basedev_suspended = 1; 7650 } 7651 } 7652 7653 /* Ensure all interrupts are disabled, e.g. if warmbooting */ 7654 dev_irq_mask_set(sinfo, 0); 7655 7656 /* Register interrupt handler */ 7657 kernel_bde->interrupt_connect(sinfo->dev_no | LKBDE_ISR2_DEV, 7658 bkn_isr, sinfo); 7659 7660 /* Init DCBs */ 7661 bkn_init_dcbs(sinfo); 7662 7663 bkn_dma_init(sinfo); 7664 7665 spin_unlock_irqrestore(&sinfo->lock, flags); 7666 7667 return sizeof(kcom_msg_hdr_t); 7668 } 7669 7670 static int 7671 bkn_knet_detach(kcom_msg_detach_t *kmsg, int len) 7672 { 7673 bkn_switch_info_t *sinfo; 7674 unsigned long flags; 7675 bkn_evt_resource_t *evt; 7676 7677 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7678 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7679 if (sinfo == NULL) { 7680 kmsg->hdr.status = KCOM_E_PARAM; 7681 return sizeof(kcom_msg_hdr_t); 7682 } 7683 7684 spin_lock_irqsave(&sinfo->lock, flags); 7685 if (sinfo->evt_idx != -1) { 7686 /* Create dummy event to unblock pending IOCTL */ 7687 sinfo->dma_events |= KCOM_DMA_INFO_F_TX_DONE; 7688 evt = &_bkn_evt[sinfo->evt_idx]; 7689 evt->evt_wq_put++; 7690 wake_up_interruptible(&evt->evt_wq); 7691 } 7692 spin_unlock_irqrestore(&sinfo->lock, flags); 7693 7694 /* Ensure that we return a valid unit number */ 7695 kmsg->hdr.unit = sinfo->dev_no; 7696 7697 return sizeof(kcom_msg_detach_t); 7698 } 7699 7700 static int 7701 bkn_knet_reprobe(kcom_msg_reprobe_t *kmsg, int len) 7702 { 7703 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7704 7705 /* Support pci hot plug and multiple instance */ 7706 if ((bkn_knet_dev_reprobe() < 0) || 7707 (bkn_knet_dev_inst_set(kmsg) < 0)) { 7708 kmsg->hdr.status = KCOM_E_RESOURCE; 7709 return sizeof(kcom_msg_reprobe_t); 7710 } 7711 7712 return sizeof(kcom_msg_reprobe_t); 7713 } 7714 7715 static int 7716 bkn_knet_netif_create(kcom_msg_netif_create_t *kmsg, int len) 7717 { 7718 bkn_switch_info_t *sinfo; 7719 struct net_device *dev; 7720 struct list_head *list; 7721 bkn_priv_t *priv, *lpriv; 7722 unsigned long flags; 7723 int found, id; 7724 uint8_t *ma; 7725 7726 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7727 7728 switch (kmsg->netif.type) { 7729 case KCOM_NETIF_T_VLAN: 7730 case KCOM_NETIF_T_PORT: 7731 case KCOM_NETIF_T_META: 7732 break; 7733 default: 7734 kmsg->hdr.status = KCOM_E_PARAM; 7735 return sizeof(kcom_msg_hdr_t); 7736 } 7737 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7738 if (sinfo == NULL) { 7739 kmsg->hdr.status = KCOM_E_PARAM; 7740 return sizeof(kcom_msg_hdr_t); 7741 } 7742 ma = kmsg->netif.macaddr; 7743 if ((ma[0] | ma[1] | ma[2] | ma[3] | ma[4] | ma[5]) == 0) { 7744 bkn_dev_mac[5]++; 7745 ma = bkn_dev_mac; 7746 } 7747 if ((dev = bkn_init_ndev(ma, kmsg->netif.name)) == NULL) { 7748 kmsg->hdr.status = KCOM_E_RESOURCE; 7749 return sizeof(kcom_msg_hdr_t); 7750 } 7751 priv = netdev_priv(dev); 7752 priv->dev = dev; 7753 priv->sinfo = sinfo; 7754 priv->type = kmsg->netif.type; 7755 priv->vlan = kmsg->netif.vlan; 7756 /* System headers are prepared at BCM API for Dune headers */ 7757 if (device_is_sand(sinfo)) { 7758 int idx = 0; 7759 for (idx = 0; idx < KCOM_NETIF_SYSTEM_HEADERS_SIZE_MAX; idx++) 7760 { 7761 priv->system_headers[idx] = kmsg->netif.system_headers[idx]; 7762 } 7763 priv->system_headers_size = kmsg->netif.system_headers_size; 7764 } 7765 if (priv->type == KCOM_NETIF_T_PORT) { 7766 priv->port = kmsg->netif.port; 7767 priv->phys_port = kmsg->netif.phys_port; 7768 priv->qnum = kmsg->netif.qnum; 7769 } else { 7770 if (device_is_sand(sinfo) && (priv->type == KCOM_NETIF_T_VLAN)) { 7771 /* PTCH.SSPA */ 7772 priv->port = kmsg->netif.port; 7773 priv->qnum = kmsg->netif.qnum; 7774 } else { 7775 priv->port = -1; 7776 } 7777 } 7778 priv->flags = kmsg->netif.flags; 7779 priv->cb_user_data = kmsg->netif.cb_user_data; 7780 7781 /* Force RCPU encapsulation if rcpu_mode */ 7782 if (rcpu_mode) { 7783 priv->flags |= KCOM_NETIF_F_RCPU_ENCAP; 7784 DBG_RCPU(("RCPU auto-enabled\n")); 7785 } 7786 7787 /* Prevent (incorrect) compiler warning */ 7788 lpriv = NULL; 7789 7790 spin_lock_irqsave(&sinfo->lock, flags); 7791 7792 /* 7793 * We insert network interfaces sorted by ID. 7794 * In case an interface is destroyed, we reuse the ID 7795 * the next time an interface is created. 7796 */ 7797 found = 0; 7798 id = 1; 7799 list_for_each(list, &sinfo->ndev_list) { 7800 lpriv = (bkn_priv_t *)list; 7801 if (id < lpriv->id) { 7802 found = 1; 7803 break; 7804 } 7805 id = lpriv->id + 1; 7806 } 7807 priv->id = id; 7808 if (found) { 7809 /* Replace previously removed interface */ 7810 list_add_tail(&priv->list, &lpriv->list); 7811 } else { 7812 /* No holes - add to end of list */ 7813 list_add_tail(&priv->list, &sinfo->ndev_list); 7814 } 7815 7816 if (id < sinfo->ndev_max) { 7817 DBG_NDEV(("Add netif ID %d to table\n", id)); 7818 sinfo->ndevs[id] = dev; 7819 } else { 7820 int ndev_max = sinfo->ndev_max + NDEVS_CHUNK; 7821 int size = ndev_max * sizeof(struct net_device *); 7822 void *ndevs = kmalloc(size, GFP_ATOMIC); 7823 if (ndevs != NULL) { 7824 DBG_NDEV(("Reallocate netif table for ID %d\n", id)); 7825 memset(ndevs, 0, size); 7826 if (sinfo->ndevs != NULL) { 7827 size = sinfo->ndev_max * sizeof(struct net_device *); 7828 memcpy(ndevs, sinfo->ndevs, size); 7829 kfree(sinfo->ndevs); 7830 } 7831 sinfo->ndevs = ndevs; 7832 sinfo->ndev_max = ndev_max; 7833 sinfo->ndevs[id] = dev; 7834 } 7835 } 7836 7837 spin_unlock_irqrestore(&sinfo->lock, flags); 7838 7839 DBG_VERB(("Assigned ID %d to Ethernet device %s\n", 7840 priv->id, dev->name)); 7841 7842 kmsg->netif.id = priv->id; 7843 memcpy(kmsg->netif.macaddr, dev->dev_addr, 6); 7844 memcpy(kmsg->netif.name, dev->name, KCOM_NETIF_NAME_MAX - 1); 7845 7846 if (device_is_sand(sinfo)) { 7847 int idx = 0; 7848 for (idx = 0; idx < priv->system_headers_size; idx++) { 7849 DBG_DUNE(("System Header[%d]: 0x%02x\n", idx, priv->system_headers[idx])); 7850 } 7851 } 7852 7853 return sizeof(*kmsg); 7854 } 7855 7856 static int 7857 bkn_knet_netif_destroy(kcom_msg_netif_destroy_t *kmsg, int len) 7858 { 7859 bkn_switch_info_t *sinfo; 7860 struct net_device *dev; 7861 bkn_priv_t *priv; 7862 struct list_head *list; 7863 unsigned long flags; 7864 int found; 7865 7866 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7867 7868 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7869 if (sinfo == NULL) { 7870 kmsg->hdr.status = KCOM_E_PARAM; 7871 return sizeof(kcom_msg_hdr_t); 7872 } 7873 7874 spin_lock_irqsave(&sinfo->lock, flags); 7875 7876 found = 0; 7877 list_for_each(list, &sinfo->ndev_list) { 7878 priv = (bkn_priv_t *)list; 7879 if (kmsg->hdr.id == priv->id) { 7880 found = 1; 7881 break; 7882 } 7883 } 7884 7885 if (!found) { 7886 spin_unlock_irqrestore(&sinfo->lock, flags); 7887 kmsg->hdr.status = KCOM_E_NOT_FOUND; 7888 return sizeof(kcom_msg_hdr_t); 7889 } 7890 7891 list_del(&priv->list); 7892 7893 if (priv->id < sinfo->ndev_max) { 7894 sinfo->ndevs[priv->id] = NULL; 7895 } 7896 7897 spin_unlock_irqrestore(&sinfo->lock, flags); 7898 7899 dev = priv->dev; 7900 DBG_VERB(("Removing virtual Ethernet device %s (%d).\n", 7901 dev->name, priv->id)); 7902 unregister_netdev(dev); 7903 free_netdev(dev); 7904 7905 return sizeof(kcom_msg_hdr_t); 7906 } 7907 7908 static int 7909 bkn_knet_netif_list(kcom_msg_netif_list_t *kmsg, int len) 7910 { 7911 bkn_switch_info_t *sinfo; 7912 bkn_priv_t *priv; 7913 struct list_head *list; 7914 unsigned long flags; 7915 int idx; 7916 7917 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7918 7919 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7920 if (sinfo == NULL) { 7921 kmsg->hdr.status = KCOM_E_PARAM; 7922 return sizeof(kcom_msg_hdr_t); 7923 } 7924 7925 spin_lock_irqsave(&sinfo->lock, flags); 7926 7927 idx = 0; 7928 list_for_each(list, &sinfo->ndev_list) { 7929 if (idx >= KCOM_NETIF_MAX) { 7930 DBG_WARN(("Too many network interfaces to list (max %d).\n", 7931 KCOM_NETIF_MAX)); 7932 break; 7933 } 7934 priv = (bkn_priv_t *)list; 7935 kmsg->id[idx] = priv->id; 7936 idx++; 7937 } 7938 kmsg->ifcnt = idx; 7939 7940 spin_unlock_irqrestore(&sinfo->lock, flags); 7941 7942 return sizeof(*kmsg) - sizeof(kmsg->id) + (idx * sizeof(kmsg->id[0])); 7943 } 7944 7945 static int 7946 bkn_knet_netif_get(kcom_msg_netif_get_t *kmsg, int len) 7947 { 7948 bkn_switch_info_t *sinfo; 7949 bkn_priv_t *priv; 7950 unsigned long flags; 7951 7952 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7953 7954 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7955 if (sinfo == NULL) { 7956 kmsg->hdr.status = KCOM_E_PARAM; 7957 return sizeof(kcom_msg_hdr_t); 7958 } 7959 7960 spin_lock_irqsave(&sinfo->lock, flags); 7961 7962 priv = bkn_netif_lookup(sinfo, kmsg->hdr.id); 7963 7964 if (priv == NULL) { 7965 spin_unlock_irqrestore(&sinfo->lock, flags); 7966 kmsg->hdr.status = KCOM_E_NOT_FOUND; 7967 return sizeof(kcom_msg_hdr_t); 7968 } 7969 7970 memcpy(kmsg->netif.macaddr, priv->dev->dev_addr, 6); 7971 memcpy(kmsg->netif.name, priv->dev->name, KCOM_NETIF_NAME_MAX - 1); 7972 kmsg->netif.vlan = priv->vlan; 7973 kmsg->netif.type = priv->type; 7974 kmsg->netif.id = priv->id; 7975 kmsg->netif.flags = priv->flags; 7976 kmsg->netif.cb_user_data = priv->cb_user_data; 7977 7978 if (priv->port < 0) { 7979 kmsg->netif.port = 0; 7980 } else { 7981 kmsg->netif.port = priv->port; 7982 } 7983 kmsg->netif.qnum = priv->qnum; 7984 7985 spin_unlock_irqrestore(&sinfo->lock, flags); 7986 7987 return sizeof(*kmsg); 7988 } 7989 7990 static int 7991 bkn_knet_filter_create(kcom_msg_filter_create_t *kmsg, int len) 7992 { 7993 bkn_switch_info_t *sinfo; 7994 struct list_head *list; 7995 bkn_filter_t *filter, *lfilter; 7996 unsigned long flags; 7997 int found, id; 7998 7999 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 8000 8001 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 8002 if (sinfo == NULL) { 8003 kmsg->hdr.status = KCOM_E_PARAM; 8004 return sizeof(kcom_msg_hdr_t); 8005 } 8006 8007 switch (kmsg->filter.type) { 8008 case KCOM_FILTER_T_RX_PKT: 8009 break; 8010 default: 8011 kmsg->hdr.status = KCOM_E_PARAM; 8012 return sizeof(kcom_msg_hdr_t); 8013 } 8014 8015 spin_lock_irqsave(&sinfo->lock, flags); 8016 8017 /* 8018 * Find available ID 8019 */ 8020 found = 1; 8021 id = 0; 8022 while (found && ++id < KCOM_FILTER_MAX) { 8023 found = 0; 8024 list_for_each(list, &sinfo->rxpf_list) { 8025 lfilter = (bkn_filter_t *)list; 8026 if (id == lfilter->kf.id) { 8027 found = 1; 8028 break; 8029 } 8030 } 8031 } 8032 if (found) { 8033 /* Too many filters */ 8034 spin_unlock_irqrestore(&sinfo->lock, flags); 8035 kmsg->hdr.status = KCOM_E_RESOURCE; 8036 return sizeof(kcom_msg_hdr_t); 8037 } 8038 filter = kmalloc(sizeof(*filter), GFP_ATOMIC); 8039 if (filter == NULL) { 8040 spin_unlock_irqrestore(&sinfo->lock, flags); 8041 kmsg->hdr.status = KCOM_E_PARAM; 8042 return sizeof(kcom_msg_hdr_t); 8043 } 8044 memset(filter, 0, sizeof(*filter)); 8045 memcpy(&filter->kf, &kmsg->filter, sizeof(filter->kf)); 8046 filter->kf.id = id; 8047 8048 /* Add according to priority */ 8049 found = 0; 8050 list_for_each(list, &sinfo->rxpf_list) { 8051 lfilter = (bkn_filter_t *)list; 8052 if (filter->kf.priority < lfilter->kf.priority) { 8053 list_add_tail(&filter->list, &lfilter->list); 8054 found = 1; 8055 break; 8056 } 8057 } 8058 if (!found) { 8059 list_add_tail(&filter->list, &sinfo->rxpf_list); 8060 } 8061 8062 kmsg->filter.id = filter->kf.id; 8063 8064 spin_unlock_irqrestore(&sinfo->lock, flags); 8065 8066 DBG_VERB(("Created filter ID %d (%s).\n", 8067 filter->kf.id, filter->kf.desc)); 8068 if (device_is_sand(sinfo)) { 8069 int idx, wsize; 8070 wsize = BYTES2WORDS(filter->kf.oob_data_size + filter->kf.pkt_data_size); 8071 DBG_DUNE(("Filter: oob_data_size = %d pkt_data_size=%d wsize %d\n", filter->kf.oob_data_size, filter->kf.pkt_data_size, wsize)); 8072 for (idx = 0; idx < wsize; idx++) 8073 { 8074 DBG_DUNE(("OOB[%d]: 0x%08x [0x%08x]\n", idx, filter->kf.data.w[idx], filter->kf.mask.w[idx])); 8075 } 8076 DBG_DUNE(("DNX system headers parameters:LB_KEY_EXT %d, STK_EXT %d, PPH_BASE %d, LIF_EXT %d %d %d, UDH_ENA %d, %d %d %d %d\n", 8077 sinfo->ftmh_lb_key_ext_size, sinfo->ftmh_stacking_ext_size, sinfo->pph_base_size, 8078 sinfo->pph_lif_ext_size[1],sinfo->pph_lif_ext_size[2], sinfo->pph_lif_ext_size[3], 8079 sinfo->udh_enable, sinfo->udh_length_type[0], sinfo->udh_length_type[1], sinfo->udh_length_type[2], sinfo->udh_length_type[3])); 8080 } 8081 return len; 8082 } 8083 8084 static int 8085 bkn_knet_filter_destroy(kcom_msg_filter_destroy_t *kmsg, int len) 8086 { 8087 bkn_switch_info_t *sinfo; 8088 bkn_filter_t *filter; 8089 struct list_head *list; 8090 unsigned long flags; 8091 int found; 8092 8093 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 8094 8095 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 8096 if (sinfo == NULL) { 8097 kmsg->hdr.status = KCOM_E_PARAM; 8098 return sizeof(kcom_msg_hdr_t); 8099 } 8100 8101 spin_lock_irqsave(&sinfo->lock, flags); 8102 8103 found = 0; 8104 list_for_each(list, &sinfo->rxpf_list) { 8105 filter = (bkn_filter_t *)list; 8106 if (kmsg->hdr.id == filter->kf.id) { 8107 found = 1; 8108 break; 8109 } 8110 } 8111 8112 if (!found) { 8113 spin_unlock_irqrestore(&sinfo->lock, flags); 8114 kmsg->hdr.status = KCOM_E_NOT_FOUND; 8115 return sizeof(kcom_msg_hdr_t); 8116 } 8117 8118 list_del(&filter->list); 8119 8120 spin_unlock_irqrestore(&sinfo->lock, flags); 8121 8122 DBG_VERB(("Removing filter ID %d.\n", filter->kf.id)); 8123 kfree(filter); 8124 8125 return sizeof(kcom_msg_hdr_t); 8126 } 8127 8128 static int 8129 bkn_knet_filter_list(kcom_msg_filter_list_t *kmsg, int len) 8130 { 8131 bkn_switch_info_t *sinfo; 8132 bkn_filter_t *filter; 8133 struct list_head *list; 8134 unsigned long flags; 8135 int idx; 8136 8137 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 8138 8139 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 8140 if (sinfo == NULL) { 8141 kmsg->hdr.status = KCOM_E_PARAM; 8142 return sizeof(kcom_msg_hdr_t); 8143 } 8144 8145 spin_lock_irqsave(&sinfo->lock, flags); 8146 8147 idx = 0; 8148 list_for_each(list, &sinfo->rxpf_list) { 8149 if (idx >= KCOM_FILTER_MAX) { 8150 DBG_WARN(("Too many filters to list (max %d).\n", 8151 KCOM_FILTER_MAX)); 8152 break; 8153 } 8154 filter = (bkn_filter_t *)list; 8155 kmsg->id[idx] = filter->kf.id; 8156 idx++; 8157 } 8158 kmsg->fcnt = idx; 8159 8160 spin_unlock_irqrestore(&sinfo->lock, flags); 8161 return sizeof(*kmsg) - sizeof(kmsg->id) + (idx * sizeof(kmsg->id[0])); 8162 } 8163 8164 static int 8165 bkn_knet_filter_get(kcom_msg_filter_get_t *kmsg, int len) 8166 { 8167 bkn_switch_info_t *sinfo; 8168 bkn_filter_t *filter; 8169 struct list_head *list; 8170 unsigned long flags; 8171 int found; 8172 8173 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 8174 8175 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 8176 if (sinfo == NULL) { 8177 kmsg->hdr.status = KCOM_E_PARAM; 8178 return sizeof(kcom_msg_hdr_t); 8179 } 8180 8181 spin_lock_irqsave(&sinfo->lock, flags); 8182 8183 found = 0; 8184 list_for_each(list, &sinfo->rxpf_list) { 8185 filter = (bkn_filter_t *)list; 8186 if (kmsg->hdr.id == filter->kf.id) { 8187 found = 1; 8188 break; 8189 } 8190 } 8191 8192 if (!found) { 8193 spin_unlock_irqrestore(&sinfo->lock, flags); 8194 kmsg->hdr.status = KCOM_E_NOT_FOUND; 8195 return sizeof(kcom_msg_hdr_t); 8196 } 8197 8198 memcpy(&kmsg->filter, &filter->kf, sizeof(kmsg->filter)); 8199 8200 spin_unlock_irqrestore(&sinfo->lock, flags); 8201 8202 return sizeof(*kmsg); 8203 } 8204 8205 static int 8206 bkn_knet_dbg_pkt_set(kcom_msg_dbg_pkt_set_t *kmsg, int len) 8207 { 8208 dbg_pkt_enable = kmsg->enable; 8209 return sizeof(kcom_msg_dbg_pkt_set_t); 8210 } 8211 8212 static int 8213 bkn_knet_dbg_pkt_get(kcom_msg_dbg_pkt_get_t *kmsg, int len) 8214 { 8215 kmsg->value = dbg_pkt_enable; 8216 return sizeof(kcom_msg_dbg_pkt_get_t); 8217 } 8218 8219 static int 8220 bkn_knet_wb_cleanup(kcom_msg_wb_cleanup_t *kmsg, int len) 8221 { 8222 bkn_switch_info_t *sinfo; 8223 bkn_dcb_chain_t *dcb_chain; 8224 unsigned long flags; 8225 int chan; 8226 8227 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 8228 8229 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 8230 if (sinfo == NULL) { 8231 kmsg->hdr.status = KCOM_E_PARAM; 8232 return sizeof(kcom_msg_hdr_t); 8233 } 8234 8235 spin_lock_irqsave(&sinfo->lock, flags); 8236 8237 for (chan = 0; chan < sinfo->rx_chans; chan++) { 8238 if (sinfo->rx[chan].api_dcb_chain) { 8239 DBG_DCB_RX(("Freeing active Rx%d DCB chain.\n", chan)); 8240 kfree(sinfo->rx[chan].api_dcb_chain); 8241 sinfo->rx[chan].api_dcb_chain = NULL; 8242 } 8243 while (!list_empty(&sinfo->rx[chan].api_dcb_list)) { 8244 dcb_chain = list_entry(sinfo->rx[chan].api_dcb_list.next, 8245 bkn_dcb_chain_t, list); 8246 list_del(&dcb_chain->list); 8247 DBG_DCB_RX(("Freeing Rx%d DCB chain.\n", chan)); 8248 kfree(dcb_chain); 8249 } 8250 sinfo->rx[chan].api_dcb_chain_end = NULL; 8251 sinfo->rx[chan].api_active = 0; 8252 } 8253 8254 spin_unlock_irqrestore(&sinfo->lock, flags); 8255 8256 return sizeof(kcom_msg_hdr_t); 8257 } 8258 8259 static int 8260 bkn_handle_cmd_req(kcom_msg_t *kmsg, int len) 8261 { 8262 /* Silently drop events and unrecognized message types */ 8263 if (kmsg->hdr.type != KCOM_MSG_TYPE_CMD) { 8264 if (kmsg->hdr.opcode == KCOM_M_STRING) { 8265 DBG_VERB(("Debug string: '%s'\n", kmsg->string.val)); 8266 return 0; 8267 } 8268 DBG_WARN(("Unsupported message (type=%d, opcode=%d)\n", 8269 kmsg->hdr.type, kmsg->hdr.opcode)); 8270 return 0; 8271 } 8272 8273 switch (kmsg->hdr.opcode) { 8274 case KCOM_M_DMA_INFO: 8275 DBG_CMD(("KCOM_M_DMA_INFO\n")); 8276 /* Packet buffer */ 8277 len = bkn_knet_dma_info(&kmsg->dma_info, len); 8278 break; 8279 case KCOM_M_VERSION: 8280 DBG_CMD(("KCOM_M_VERSION\n")); 8281 /* Return procotol version */ 8282 len = bkn_knet_version(&kmsg->version, len); 8283 break; 8284 case KCOM_M_HW_RESET: 8285 DBG_CMD(("KCOM_M_HW_RESET\n")); 8286 /* Shut down DMA and release buffers */ 8287 len = bkn_knet_hw_reset(&kmsg->hw_reset, len); 8288 break; 8289 case KCOM_M_HW_INIT: 8290 DBG_CMD(("KCOM_M_HW_INIT\n")); 8291 /* Initialize DMA */ 8292 len = bkn_knet_hw_init(&kmsg->hw_init, len); 8293 break; 8294 case KCOM_M_DETACH: 8295 DBG_CMD(("KCOM_M_DETACH\n")); 8296 /* Detach kernel module */ 8297 len = bkn_knet_detach(&kmsg->detach, len); 8298 break; 8299 case KCOM_M_REPROBE: 8300 DBG_CMD(("KCOM_M_REPROBE\n")); 8301 /* Reprobe device */ 8302 len = bkn_knet_reprobe(&kmsg->reprobe, len); 8303 break; 8304 case KCOM_M_NETIF_CREATE: 8305 DBG_CMD(("KCOM_M_NETIF_CREATE\n")); 8306 /* Create network interface */ 8307 len = bkn_knet_netif_create(&kmsg->netif_create, len); 8308 break; 8309 case KCOM_M_NETIF_DESTROY: 8310 DBG_CMD(("KCOM_M_NETIF_DESTROY\n")); 8311 /* Destroy network interface */ 8312 len = bkn_knet_netif_destroy(&kmsg->netif_destroy, len); 8313 break; 8314 case KCOM_M_NETIF_LIST: 8315 DBG_CMD(("KCOM_M_NETIF_LIST\n")); 8316 /* Return list of IDs of installed network interfaces */ 8317 len = bkn_knet_netif_list(&kmsg->netif_list, len); 8318 break; 8319 case KCOM_M_NETIF_GET: 8320 DBG_CMD(("KCOM_M_NETIF_GET\n")); 8321 /* Return network interface info */ 8322 len = bkn_knet_netif_get(&kmsg->netif_get, len); 8323 break; 8324 case KCOM_M_FILTER_CREATE: 8325 DBG_CMD(("KCOM_M_FILTER_CREATE\n")); 8326 /* Create packet filter */ 8327 len = bkn_knet_filter_create(&kmsg->filter_create, len); 8328 break; 8329 case KCOM_M_FILTER_DESTROY: 8330 DBG_CMD(("KCOM_M_FILTER_DESTROY\n")); 8331 /* Destroy packet filter */ 8332 len = bkn_knet_filter_destroy(&kmsg->filter_destroy, len); 8333 break; 8334 case KCOM_M_FILTER_LIST: 8335 DBG_CMD(("KCOM_M_FILTER_LIST\n")); 8336 /* Return list of IDs of installed packet filters */ 8337 len = bkn_knet_filter_list(&kmsg->filter_list, len); 8338 break; 8339 case KCOM_M_FILTER_GET: 8340 DBG_CMD(("KCOM_M_FILTER_GET\n")); 8341 /* Return packet filter info */ 8342 len = bkn_knet_filter_get(&kmsg->filter_get, len); 8343 break; 8344 case KCOM_M_DBGPKT_SET: 8345 DBG_CMD(("KCOM_M_DBGPKT_SET\n")); 8346 /* Set debugging packet function */ 8347 len = bkn_knet_dbg_pkt_set(&kmsg->dbg_pkt_set, len); 8348 break; 8349 case KCOM_M_DBGPKT_GET: 8350 DBG_CMD(("KCOM_M_DBGPKT_GET\n")); 8351 /* Get debugging packet function info */ 8352 len = bkn_knet_dbg_pkt_get(&kmsg->dbg_pkt_get, len); 8353 break; 8354 case KCOM_M_WB_CLEANUP: 8355 DBG_CMD(("KCOM_M_WB_CLEANUP\n")); 8356 /* Clean up for warmbooting */ 8357 len = bkn_knet_wb_cleanup(&kmsg->wb_cleanup, len); 8358 break; 8359 case KCOM_M_CLOCK_CMD: 8360 /* PHC clock control*/ 8361 if (knet_hw_tstamp_ioctl_cmd_cb) { 8362 bkn_switch_info_t *sinfo; 8363 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 8364 if (sinfo == NULL) { 8365 /* The device is not probed or initialized yet.*/ 8366 return 0; 8367 } 8368 DBG_CMD(("KCOM_M_CLOCK_CMD\n")); 8369 len = knet_hw_tstamp_ioctl_cmd_cb(&kmsg->clock_cmd, len, sinfo->dcb_type); 8370 } else { 8371 DBG_WARN(("Unsupported command (type=%d, opcode=%d)\n", 8372 kmsg->hdr.type, kmsg->hdr.opcode)); 8373 kmsg->hdr.opcode = 0; 8374 len = sizeof(kcom_msg_hdr_t); 8375 } 8376 break; 8377 default: 8378 DBG_WARN(("Unsupported command (type=%d, opcode=%d)\n", 8379 kmsg->hdr.type, kmsg->hdr.opcode)); 8380 kmsg->hdr.opcode = 0; 8381 len = sizeof(kcom_msg_hdr_t); 8382 break; 8383 } 8384 return len; 8385 } 8386 8387 static int 8388 bkn_get_next_dma_event(kcom_msg_dma_info_t *kmsg) 8389 { 8390 static int last_dev_no = 0; 8391 bkn_switch_info_t *sinfo; 8392 unsigned long flags; 8393 int dev_no, dev_evt; 8394 bkn_evt_resource_t *evt; 8395 8396 dev_evt = kmsg->hdr.unit; 8397 sinfo = bkn_sinfo_from_unit(dev_evt); 8398 if (sinfo == NULL) { 8399 /* The device is not probed or initialized yet.*/ 8400 return 0; 8401 } 8402 if (sinfo->evt_idx == -1) { 8403 /* Event queue is not ready yet */ 8404 return 0; 8405 } 8406 8407 DBG_INST(("%s dev %d evt_idx %d\n",__FUNCTION__, dev_evt, sinfo->evt_idx)); 8408 dev_no = last_dev_no; 8409 while (1) { 8410 dev_no++; 8411 sinfo = bkn_sinfo_from_unit(dev_no); 8412 if (sinfo == NULL) { 8413 dev_no = 0; 8414 sinfo = bkn_sinfo_from_unit(dev_no); 8415 } 8416 8417 if (sinfo && (sinfo->inst_id != 0) && 8418 ((sinfo->inst_id & (1 << dev_evt)) == 0)) { 8419 DBG_INST((" %s skip dev(%d)\n",__FUNCTION__,dev_no)); 8420 continue; 8421 } 8422 8423 if (sinfo->evt_idx == -1) { 8424 /* Event queue is not ready yet */ 8425 continue; 8426 } 8427 if (sinfo && sinfo->dma_events) { 8428 DBG_EVT(("Next DMA events (0x%08x)\n", sinfo->dma_events)); 8429 kmsg->hdr.unit = sinfo->dev_no; 8430 8431 spin_lock_irqsave(&sinfo->lock, flags); 8432 kmsg->dma_info.flags = sinfo->dma_events; 8433 sinfo->dma_events = 0; 8434 spin_unlock_irqrestore(&sinfo->lock, flags); 8435 8436 last_dev_no = dev_no; 8437 break; 8438 } 8439 8440 if ((dev_no == last_dev_no) || 8441 (_bkn_multi_inst && (dev_no == dev_evt))) { 8442 evt = &_bkn_evt[sinfo->evt_idx]; 8443 DBG_INST(("dev_no %d dev_evt %d wait queue index %d\n", 8444 dev_no, dev_evt, sinfo->evt_idx)); 8445 wait_event_interruptible(evt->evt_wq, 8446 evt->evt_wq_get != evt->evt_wq_put); 8447 DBG_VERB(("Event thread wakeup\n")); 8448 8449 /* Thread interrupted */ 8450 if (signal_pending(current)) { 8451 return 0; 8452 } 8453 8454 evt->evt_wq_get = evt->evt_wq_put; 8455 } 8456 } 8457 return sizeof(*kmsg); 8458 } 8459 8460 static int 8461 _cleanup(void) 8462 { 8463 struct list_head *list; 8464 struct net_device *dev; 8465 bkn_filter_t *filter; 8466 bkn_priv_t *priv; 8467 bkn_switch_info_t *sinfo; 8468 unsigned long flags; 8469 8470 /* Inidicate that we are shutting down */ 8471 module_initialized = 0; 8472 8473 bkn_proc_cleanup(); 8474 remove_proc_entry("bcm/knet", NULL); 8475 remove_proc_entry("bcm", NULL); 8476 8477 list_for_each(list, &_sinfo_list) { 8478 sinfo = (bkn_switch_info_t *)list; 8479 8480 del_timer_sync(&sinfo->timer); 8481 del_timer_sync(&sinfo->rxtick); 8482 8483 spin_lock_irqsave(&sinfo->lock, flags); 8484 bkn_dma_abort(sinfo); 8485 dev_irq_mask_set(sinfo, 0); 8486 spin_unlock_irqrestore(&sinfo->lock, flags); 8487 8488 DBG_IRQ(("Unregister ISR.\n")); 8489 kernel_bde->interrupt_disconnect(sinfo->dev_no | LKBDE_ISR2_DEV); 8490 8491 if (use_napi) { 8492 while (sinfo->napi_poll_mode) { 8493 bkn_sleep(1); 8494 } 8495 } 8496 8497 spin_lock_irqsave(&sinfo->lock, flags); 8498 bkn_clean_dcbs(sinfo); 8499 skb_queue_purge(&sinfo->tx_ptp_queue); 8500 spin_unlock_irqrestore(&sinfo->lock, flags); 8501 } 8502 8503 /* Destroy all switch devices */ 8504 while (!list_empty(&_sinfo_list)) { 8505 sinfo = list_entry(_sinfo_list.next, bkn_switch_info_t, list); 8506 8507 /* Destroy all associated Rx packet filters */ 8508 while (!list_empty(&sinfo->rxpf_list)) { 8509 filter = list_entry(sinfo->rxpf_list.next, bkn_filter_t, list); 8510 list_del(&filter->list); 8511 DBG_VERB(("Removing filter ID %d.\n", filter->kf.id)); 8512 kfree(filter); 8513 } 8514 8515 /* Destroy all associated virtual net devices */ 8516 while (!list_empty(&sinfo->ndev_list)) { 8517 priv = list_entry(sinfo->ndev_list.next, bkn_priv_t, list); 8518 list_del(&priv->list); 8519 dev = priv->dev; 8520 DBG_VERB(("Removing virtual Ethernet device %s.\n", dev->name)); 8521 unregister_netdev(dev); 8522 free_netdev(dev); 8523 } 8524 if (sinfo->ndevs != NULL) { 8525 kfree(sinfo->ndevs); 8526 } 8527 8528 /* Destroy base net device */ 8529 if (sinfo->dev) { 8530 DBG_VERB(("Removing Ethernet device %s.\n", sinfo->dev->name)); 8531 unregister_netdev(sinfo->dev); 8532 free_netdev(sinfo->dev); 8533 } 8534 8535 DBG_VERB(("Removing switch device.\n")); 8536 bkn_destroy_sinfo(sinfo); 8537 } 8538 8539 return 0; 8540 } 8541 8542 static int 8543 bkn_knet_dev_reinit(int d) 8544 { 8545 bkn_switch_info_t *sinfo; 8546 uint32 dev_state; 8547 unsigned long flags; 8548 8549 if (lkbde_dev_state_get(d, &dev_state) < 0) { 8550 return -1; 8551 } 8552 DBG_VERB(("%s dev %d dev_state %d\n",__FUNCTION__, d, dev_state)); 8553 if (dev_state == BDE_DEV_STATE_CHANGED) { 8554 sinfo = bkn_sinfo_from_unit(d); 8555 spin_lock_irqsave(&sinfo->lock, flags); 8556 sinfo->base_addr = lkbde_get_dev_virt(d); 8557 sinfo->dma_dev = lkbde_get_dma_dev(d); 8558 sinfo->pdev = lkbde_get_hw_dev(d); 8559 spin_unlock_irqrestore(&sinfo->lock, flags); 8560 8561 dev_state = 0; 8562 lkbde_dev_state_set(d, dev_state); 8563 } 8564 return 0; 8565 } 8566 8567 static int 8568 bkn_knet_dev_init(int d) 8569 { 8570 uint32_t dev_type; 8571 struct net_device *dev; 8572 bkn_switch_info_t *sinfo; 8573 bkn_priv_t *priv; 8574 char *bdev_name; 8575 const ibde_dev_t *bde_dev; 8576 8577 DBG_VERB(("%s dev %d\n",__FUNCTION__, d)); 8578 /* Base network device name */ 8579 bdev_name = "bcm%d"; 8580 if (base_dev_name) { 8581 if (strlen(base_dev_name) < IFNAMSIZ) { 8582 bdev_name = base_dev_name; 8583 } else { 8584 DBG_WARN(("Base device name too long\n")); 8585 } 8586 } 8587 8588 dev_type = kernel_bde->get_dev_type(d); 8589 DBG_VERB(("Found device type 0x%x\n", dev_type)); 8590 if ((dev_type & BDE_SWITCH_DEV_TYPE) == 0) { 8591 DBG_WARN(("Not switch device - skipping\n")); 8592 return 0; 8593 } 8594 switch (dev_type & BDE_DEV_BUS_TYPE_MASK) { 8595 case BDE_PCI_DEV_TYPE: 8596 case BDE_ICS_DEV_TYPE: 8597 case BDE_AXI_DEV_TYPE: 8598 break; 8599 default: 8600 DBG_WARN(("Not PCI/ICS/AXI device - skipping\n")); 8601 return 0; 8602 } 8603 8604 if ((sinfo = bkn_create_sinfo(d)) == NULL) { 8605 _cleanup(); 8606 return -ENOMEM; 8607 } 8608 8609 /* Initialize the cpu_no.*/ 8610 if (dev_type & BDE_AXI_DEV_TYPE) { 8611 /* AXI device type implies the activated iProc iHost */ 8612 sinfo->cpu_no = 1; 8613 } 8614 /* Initialize default RCPU signature */ 8615 if ((bde_dev = kernel_bde->get_dev(d)) != NULL) { 8616 sinfo->rcpu_sig = bde_dev->device & ~0xf; 8617 } 8618 /* Check for override */ 8619 if (rcpu_signature) { 8620 sinfo->rcpu_sig = rcpu_signature; 8621 } 8622 8623 /* Create base virtual net device */ 8624 bkn_dev_mac[5]++; 8625 if ((dev = bkn_init_ndev(bkn_dev_mac, bdev_name)) == NULL) { 8626 _cleanup(); 8627 return -ENOMEM; 8628 } else { 8629 sinfo->dev = dev; 8630 priv = netdev_priv(dev); 8631 priv->dev = dev; 8632 priv->sinfo = sinfo; 8633 priv->vlan = 1; 8634 priv->port = -1; 8635 priv->id = -1; 8636 } 8637 8638 if (use_napi) { 8639 netif_napi_add(dev, &sinfo->napi, bkn_poll, napi_weight); 8640 } 8641 return 0; 8642 } 8643 8644 static int 8645 _init(void) 8646 { 8647 int idx; 8648 int num_dev; 8649 int rv; 8650 bkn_evt_resource_t *evt; 8651 8652 /* Connect to the kernel bde */ 8653 if ((linux_bde_create(NULL, &kernel_bde) < 0) || kernel_bde == NULL) { 8654 return -ENODEV; 8655 } 8656 8657 /* Randomize Lower 3 bytes of the MAC address (TESTING ONLY) */ 8658 get_random_bytes(&bkn_dev_mac[3], 3); 8659 8660 /* Check for user-supplied MAC address (recommended) */ 8661 if (mac_addr != NULL && strlen(mac_addr) == 17) { 8662 for (idx = 0; idx < 6; idx++) { 8663 bkn_dev_mac[idx] = simple_strtoul(&mac_addr[idx*3], NULL, 16); 8664 } 8665 /* Do not allow multicast address */ 8666 bkn_dev_mac[0] &= ~0x01; 8667 } 8668 8669 /* Optional RCPU MAC addresses */ 8670 if (rcpu_dmac != NULL && strlen(rcpu_dmac) == 17) { 8671 for (idx = 0; idx < 6; idx++) { 8672 bkn_rcpu_dmac[idx] = simple_strtoul(&rcpu_dmac[idx*3], NULL, 16); 8673 } 8674 } 8675 if (rcpu_smac != NULL && strlen(rcpu_smac) == 17) { 8676 for (idx = 0; idx < 6; idx++) { 8677 bkn_rcpu_smac[idx] = simple_strtoul(&rcpu_smac[idx*3], NULL, 16); 8678 } 8679 } 8680 8681 /* NAPI implies that base device must be up before we can pass traffic */ 8682 if (use_napi) { 8683 basedev_suspend = 1; 8684 } 8685 8686 num_dev = kernel_bde->num_devices(BDE_ALL_DEVICES); 8687 for (idx = 0; idx < num_dev; idx++) { 8688 rv = bkn_knet_dev_init(idx); 8689 if (rv) { 8690 return rv; 8691 } 8692 } 8693 8694 /* Initialize proc files */ 8695 proc_mkdir("bcm", NULL); 8696 bkn_proc_root = proc_mkdir("bcm/knet", NULL); 8697 8698 bkn_proc_init(); 8699 8700 /* Initialize event queue */ 8701 for (idx = 0; idx < LINUX_BDE_MAX_DEVICES; idx++) { 8702 memset(&_bkn_evt[idx], 0, sizeof(bkn_evt_resource_t)); 8703 } 8704 evt = &_bkn_evt[0]; 8705 init_waitqueue_head(&evt->evt_wq); 8706 8707 module_initialized = 1; 8708 8709 return 0; 8710 } 8711 8712 static int 8713 _ioctl(unsigned int cmd, unsigned long arg) 8714 { 8715 bkn_ioctl_t io; 8716 kcom_msg_t kmsg; 8717 8718 if (!module_initialized) { 8719 return -EFAULT; 8720 } 8721 8722 if (copy_from_user(&io, (void*)arg, sizeof(io))) { 8723 return -EFAULT; 8724 } 8725 8726 if (io.len > sizeof(kmsg)) { 8727 return -EINVAL; 8728 } 8729 8730 io.rc = 0; 8731 8732 switch(cmd) { 8733 case 0: 8734 if (io.len > 0) { 8735 if (copy_from_user(&kmsg, (void *)(unsigned long)io.buf, io.len)) { 8736 return -EFAULT; 8737 } 8738 ioctl_cmd++; 8739 io.len = bkn_handle_cmd_req(&kmsg, io.len); 8740 ioctl_cmd--; 8741 } else { 8742 memset(&kmsg, 0, sizeof(kcom_msg_t)); 8743 /* 8744 * Retrive the kmsg.hdr.unit from user space. The dma event queue 8745 * selection is based the instance derived from unit. 8746 */ 8747 if (copy_from_user(&kmsg, (void *)(unsigned long)io.buf, sizeof(kmsg))) { 8748 return -EFAULT; 8749 } 8750 kmsg.hdr.type = KCOM_MSG_TYPE_EVT; 8751 kmsg.hdr.opcode = KCOM_M_DMA_INFO; 8752 ioctl_evt++; 8753 io.len = bkn_get_next_dma_event((kcom_msg_dma_info_t *)&kmsg); 8754 ioctl_evt--; 8755 } 8756 if (io.len > 0) { 8757 if (copy_to_user((void *)(unsigned long)io.buf, &kmsg, io.len)) { 8758 return -EFAULT; 8759 } 8760 } 8761 break; 8762 default: 8763 gprintk("Invalid IOCTL"); 8764 io.rc = -1; 8765 break; 8766 } 8767 8768 if (copy_to_user((void*)arg, &io, sizeof(io))) { 8769 return -EFAULT; 8770 } 8771 8772 return 0; 8773 } 8774 8775 static gmodule_t _gmodule = { 8776 name: MODULE_NAME, 8777 major: MODULE_MAJOR, 8778 init: _init, 8779 cleanup: _cleanup, 8780 pprint: _pprint, 8781 ioctl: _ioctl, 8782 open: NULL, 8783 close: NULL, 8784 }; 8785 8786 gmodule_t * 8787 gmodule_get(void) 8788 { 8789 EXPORT_NO_SYMBOLS; 8790 return &_gmodule; 8791 } 8792 8793 /* 8794 * Call-back interfaces for other Linux kernel drivers. 8795 * 8796 * The Rx call-back allows an external module to modify SKB contents 8797 * before it is handed off to the Linux network stack. 8798 * 8799 * The Tx call-back allows an external module to modify SKB contents 8800 * before it is injected inot the switch. 8801 * 8802 * The HW timestamp callbacks invoke an external module to enable, disable 8803 * HW timestamp on a specific port which is indicated while the netif is 8804 * created through KNET API. KNET can also get device-specific SOBMH and 8805 * timestamp for a Tx PTP packet through these callbacks. 8806 */ 8807 8808 int 8809 bkn_rx_skb_cb_register(knet_skb_cb_f rx_cb) 8810 { 8811 if (knet_rx_cb != NULL) { 8812 return -1; 8813 } 8814 knet_rx_cb = rx_cb; 8815 return 0; 8816 } 8817 8818 int 8819 bkn_rx_skb_cb_unregister(knet_skb_cb_f rx_cb) 8820 { 8821 if (rx_cb != NULL && knet_rx_cb != rx_cb) { 8822 return -1; 8823 } 8824 knet_rx_cb = NULL; 8825 return 0; 8826 } 8827 8828 int 8829 bkn_tx_skb_cb_register(knet_skb_cb_f tx_cb) 8830 { 8831 if (knet_tx_cb != NULL) { 8832 return -1; 8833 } 8834 knet_tx_cb = tx_cb; 8835 return 0; 8836 } 8837 8838 int 8839 bkn_tx_skb_cb_unregister(knet_skb_cb_f tx_cb) 8840 { 8841 if (tx_cb != NULL && knet_tx_cb != tx_cb) { 8842 return -1; 8843 } 8844 knet_tx_cb = NULL; 8845 return 0; 8846 } 8847 8848 int 8849 bkn_filter_cb_register(knet_filter_cb_f filter_cb) 8850 { 8851 if (knet_filter_cb != NULL) { 8852 return -1; 8853 } 8854 knet_filter_cb = filter_cb; 8855 return 0; 8856 } 8857 8858 int 8859 bkn_filter_cb_unregister(knet_filter_cb_f filter_cb) 8860 { 8861 if (filter_cb != NULL && knet_filter_cb != filter_cb) { 8862 return -1; 8863 } 8864 knet_filter_cb = NULL; 8865 return 0; 8866 } 8867 8868 int 8869 bkn_hw_tstamp_enable_cb_register(knet_hw_tstamp_enable_cb_f hw_tstamp_enable_cb) 8870 { 8871 if (knet_hw_tstamp_enable_cb != NULL) { 8872 return -1; 8873 } 8874 knet_hw_tstamp_enable_cb = hw_tstamp_enable_cb; 8875 return 0; 8876 } 8877 8878 int 8879 bkn_hw_tstamp_enable_cb_unregister(knet_hw_tstamp_enable_cb_f hw_tstamp_enable_cb) 8880 { 8881 if (hw_tstamp_enable_cb == NULL || 8882 knet_hw_tstamp_enable_cb != hw_tstamp_enable_cb) { 8883 return -1; 8884 } 8885 knet_hw_tstamp_enable_cb = NULL; 8886 return 0; 8887 } 8888 8889 int 8890 bkn_hw_tstamp_disable_cb_register(knet_hw_tstamp_enable_cb_f hw_tstamp_disable_cb) 8891 { 8892 if (knet_hw_tstamp_disable_cb != NULL) { 8893 return -1; 8894 } 8895 knet_hw_tstamp_disable_cb = hw_tstamp_disable_cb; 8896 return 0; 8897 } 8898 8899 int 8900 bkn_hw_tstamp_disable_cb_unregister(knet_hw_tstamp_enable_cb_f hw_tstamp_disable_cb) 8901 { 8902 if (hw_tstamp_disable_cb == NULL || 8903 knet_hw_tstamp_disable_cb != hw_tstamp_disable_cb) { 8904 return -1; 8905 } 8906 knet_hw_tstamp_disable_cb = NULL; 8907 return 0; 8908 } 8909 8910 int 8911 bkn_hw_tstamp_tx_time_get_cb_register(knet_hw_tstamp_tx_time_get_cb_f hw_tstamp_tx_time_get_cb) 8912 { 8913 if (knet_hw_tstamp_tx_time_get_cb != NULL) { 8914 return -1; 8915 } 8916 knet_hw_tstamp_tx_time_get_cb = hw_tstamp_tx_time_get_cb; 8917 return 0; 8918 } 8919 8920 int 8921 bkn_hw_tstamp_tx_time_get_cb_unregister(knet_hw_tstamp_tx_time_get_cb_f hw_tstamp_tx_time_get_cb) 8922 { 8923 if (hw_tstamp_tx_time_get_cb == NULL || 8924 knet_hw_tstamp_tx_time_get_cb != hw_tstamp_tx_time_get_cb) { 8925 return -1; 8926 } 8927 knet_hw_tstamp_tx_time_get_cb = NULL; 8928 return 0; 8929 } 8930 8931 int 8932 bkn_hw_tstamp_tx_meta_get_cb_register(knet_hw_tstamp_tx_meta_get_cb_f hw_tstamp_tx_meta_get_cb) 8933 { 8934 if (knet_hw_tstamp_tx_meta_get_cb != NULL) { 8935 return -1; 8936 } 8937 knet_hw_tstamp_tx_meta_get_cb = hw_tstamp_tx_meta_get_cb; 8938 return 0; 8939 } 8940 8941 int 8942 bkn_hw_tstamp_tx_meta_get_cb_unregister(knet_hw_tstamp_tx_meta_get_cb_f hw_tstamp_tx_meta_get_cb) 8943 { 8944 if (hw_tstamp_tx_meta_get_cb == NULL || 8945 knet_hw_tstamp_tx_meta_get_cb != hw_tstamp_tx_meta_get_cb) { 8946 return -1; 8947 } 8948 knet_hw_tstamp_tx_meta_get_cb = NULL; 8949 return 0; 8950 } 8951 8952 int 8953 bkn_hw_tstamp_ptp_clock_index_cb_register(knet_hw_tstamp_ptp_clock_index_cb_f hw_tstamp_ptp_clock_index_cb) 8954 { 8955 if (knet_hw_tstamp_ptp_clock_index_cb != NULL) { 8956 return -1; 8957 } 8958 knet_hw_tstamp_ptp_clock_index_cb = hw_tstamp_ptp_clock_index_cb; 8959 return 0; 8960 } 8961 8962 int 8963 bkn_hw_tstamp_ptp_clock_index_cb_unregister(knet_hw_tstamp_ptp_clock_index_cb_f hw_tstamp_ptp_clock_index_cb) 8964 { 8965 if (hw_tstamp_ptp_clock_index_cb == NULL || 8966 knet_hw_tstamp_ptp_clock_index_cb != hw_tstamp_ptp_clock_index_cb) { 8967 return -1; 8968 } 8969 knet_hw_tstamp_ptp_clock_index_cb = NULL; 8970 return 0; 8971 } 8972 8973 int 8974 bkn_hw_tstamp_rx_time_upscale_cb_register(knet_hw_tstamp_rx_time_upscale_cb_f hw_tstamp_rx_time_upscale_cb) 8975 { 8976 if (knet_hw_tstamp_rx_time_upscale_cb != NULL) { 8977 return -1; 8978 } 8979 knet_hw_tstamp_rx_time_upscale_cb = hw_tstamp_rx_time_upscale_cb; 8980 return 0; 8981 } 8982 8983 int 8984 bkn_hw_tstamp_rx_time_upscale_cb_unregister(knet_hw_tstamp_rx_time_upscale_cb_f hw_tstamp_rx_time_upscale_cb) 8985 { 8986 if (hw_tstamp_rx_time_upscale_cb == NULL || 8987 knet_hw_tstamp_rx_time_upscale_cb != hw_tstamp_rx_time_upscale_cb) { 8988 return -1; 8989 } 8990 knet_hw_tstamp_rx_time_upscale_cb = NULL; 8991 return 0; 8992 } 8993 8994 int 8995 bkn_hw_tstamp_ioctl_cmd_cb_register(knet_hw_tstamp_ioctl_cmd_cb_f hw_tstamp_ioctl_cmd_cb) 8996 { 8997 if (knet_hw_tstamp_ioctl_cmd_cb != NULL) { 8998 return -1; 8999 } 9000 knet_hw_tstamp_ioctl_cmd_cb = hw_tstamp_ioctl_cmd_cb; 9001 return 0; 9002 } 9003 9004 int 9005 bkn_hw_tstamp_ioctl_cmd_cb_unregister(knet_hw_tstamp_ioctl_cmd_cb_f hw_tstamp_ioctl_cmd_cb) 9006 { 9007 if (knet_hw_tstamp_ioctl_cmd_cb == NULL || 9008 knet_hw_tstamp_ioctl_cmd_cb != hw_tstamp_ioctl_cmd_cb) { 9009 return -1; 9010 } 9011 knet_hw_tstamp_ioctl_cmd_cb = NULL; 9012 return 0; 9013 } 9014 9015 LKM_EXPORT_SYM(bkn_rx_skb_cb_register); 9016 LKM_EXPORT_SYM(bkn_rx_skb_cb_unregister); 9017 LKM_EXPORT_SYM(bkn_tx_skb_cb_register); 9018 LKM_EXPORT_SYM(bkn_tx_skb_cb_unregister); 9019 LKM_EXPORT_SYM(bkn_filter_cb_register); 9020 LKM_EXPORT_SYM(bkn_filter_cb_unregister); 9021 LKM_EXPORT_SYM(bkn_hw_tstamp_enable_cb_register); 9022 LKM_EXPORT_SYM(bkn_hw_tstamp_enable_cb_unregister); 9023 LKM_EXPORT_SYM(bkn_hw_tstamp_disable_cb_register); 9024 LKM_EXPORT_SYM(bkn_hw_tstamp_disable_cb_unregister); 9025 LKM_EXPORT_SYM(bkn_hw_tstamp_tx_time_get_cb_register); 9026 LKM_EXPORT_SYM(bkn_hw_tstamp_tx_time_get_cb_unregister); 9027 LKM_EXPORT_SYM(bkn_hw_tstamp_tx_meta_get_cb_register); 9028 LKM_EXPORT_SYM(bkn_hw_tstamp_tx_meta_get_cb_unregister); 9029 LKM_EXPORT_SYM(bkn_hw_tstamp_ptp_clock_index_cb_register); 9030 LKM_EXPORT_SYM(bkn_hw_tstamp_ptp_clock_index_cb_unregister); 9031 LKM_EXPORT_SYM(bkn_hw_tstamp_rx_time_upscale_cb_register); 9032 LKM_EXPORT_SYM(bkn_hw_tstamp_rx_time_upscale_cb_unregister); 9033 LKM_EXPORT_SYM(bkn_hw_tstamp_ioctl_cmd_cb_register); 9034 LKM_EXPORT_SYM(bkn_hw_tstamp_ioctl_cmd_cb_unregister);