bcm-knet.c (288298B)
1 /* 2 * 3 * This license is set out in https://raw.githubusercontent.com/Broadcom-Network-Switching-Software/OpenBCM/master/Legal/LICENSE file. 4 * 5 * Copyright 2007-2019 Broadcom Inc. All rights reserved. 6 */ 7 8 /* 9 * This module implements a Linux network driver for Broadcom 10 * XGS switch devices. The driver simultaneously serves a 11 * number of vitual Linux network devices and a Tx/Rx API 12 * implemented in user space. 13 * 14 * Packets received from the switch device are sent to either 15 * a virtual Linux network device or the user mode Rx API 16 * based on a set of packet filters.susp 17 * 18 * Packets from the virtual Linux network devices and the user 19 * mode Tx API are multiplexed with priority given to the Tx API. 20 * 21 * A message-based IOCTL interface is used for managing packet 22 * filters and virtual Linux network interfaces. 23 * 24 * A virtual network interface can be configured to work in RCPU 25 * mode, which means that packets from the switch device will 26 * be encasulated with a RCPU header and a block of meta data 27 * that basically contains the core DCB information. Likewise, 28 * packets received from the Linux network stack are assumed to 29 * be RCPU encapsulated when going out on an interface in RCPU 30 * mode. 31 * 32 * The module implements basic Rx DMA rate control. The rate is 33 * specified in packets per second, and different Rx DMA channels 34 * can be configured to use different maximum packet rates. 35 * The packet rate can be configure as a module parameter, and 36 * it can also be changed dynamically through the proc file 37 * system (syntax is described in function header comment). 38 * 39 * To support multiple instance, each instance has its event queue. 40 * 41 * To support pci hot-plug in this module, the resource update 42 * should be handled when the PCI device is re-plugged. 43 * NOTE: the KNET detach should be invoked befere removing the 44 * device. 45 * 46 * For a list of supported module parameters, please see below. 47 */ 48 49 #include <gmodule.h> /* Must be included first */ 50 #include <linux-bde.h> 51 #include <kcom.h> 52 #include <bcm-knet.h> 53 54 #include <linux/netdevice.h> 55 #include <linux/etherdevice.h> 56 #include <linux/ethtool.h> 57 #include <linux/random.h> 58 #include <linux/seq_file.h> 59 #include <linux/if_vlan.h> 60 61 62 MODULE_AUTHOR("Broadcom Corporation"); 63 MODULE_DESCRIPTION("Network Device Driver for Broadcom BCM TxRx API"); 64 MODULE_LICENSE("GPL"); 65 66 static int debug; 67 LKM_MOD_PARAM(debug, "i", int, 0); 68 MODULE_PARM_DESC(debug, 69 "Debug level (default 0)"); 70 71 static char *mac_addr = NULL; 72 LKM_MOD_PARAM(mac_addr, "s", charp, 0); 73 MODULE_PARM_DESC(mac_addr, 74 "Ethernet MAC address (default 02:10:18:xx:xx:xx)"); 75 76 static int rx_buffer_size = 9216; 77 LKM_MOD_PARAM(rx_buffer_size, "i", int, 0); 78 MODULE_PARM_DESC(rx_buffer_size, 79 "Size of RX packet buffers (default 9216)"); 80 81 static int default_mtu = 1500; 82 LKM_MOD_PARAM(default_mtu, "i", int, 0); 83 MODULE_PARM_DESC(default_mtu, 84 "Default MTU for KNET network interfaces (default 1500)"); 85 86 static int rx_sync_retry = 1000; 87 LKM_MOD_PARAM(rx_sync_retry, "i", int, 0); 88 MODULE_PARM_DESC(rx_sync_retry, 89 "Retries if chain is incomplete on interrupt (default 10)"); 90 91 static char *base_dev_name = NULL; 92 LKM_MOD_PARAM(base_dev_name, "s", charp, 0); 93 MODULE_PARM_DESC(base_dev_name, 94 "Base device name (default bcm0, bcm1, etc.)"); 95 96 static int rcpu_mode = 0; 97 LKM_MOD_PARAM(rcpu_mode, "i", int, 0); 98 MODULE_PARM_DESC(rcpu_mode, 99 "Enable RCPU encapsulation (default 0)"); 100 101 static char *rcpu_dmac = NULL; 102 LKM_MOD_PARAM(rcpu_dmac, "s", charp, 0); 103 MODULE_PARM_DESC(rcpu_dmac, 104 "RCPU destination MAC address (by default use L2 destination MAC address)"); 105 106 static char *rcpu_smac = NULL; 107 LKM_MOD_PARAM(rcpu_smac, "s", charp, 0); 108 MODULE_PARM_DESC(rcpu_smac, 109 "RCPU source MAC address (by default use L2 source MAC address)"); 110 111 static int rcpu_ethertype = 0xde08; 112 LKM_MOD_PARAM(rcpu_ethertype, "i", int, 0); 113 MODULE_PARM_DESC(rcpu_ethertype, 114 "RCPU EtherType (default DE08h)"); 115 116 static int rcpu_signature = 0; 117 LKM_MOD_PARAM(rcpu_signature, "i", int, 0); 118 MODULE_PARM_DESC(rcpu_signature, 119 "RCPU Signature (default is PCI device ID)"); 120 121 static int rcpu_vlan = 1; 122 LKM_MOD_PARAM(rcpu_vlan, "i", int, 0); 123 MODULE_PARM_DESC(rcpu_vlan, 124 "RCPU VLAN ID (default 1)"); 125 126 static int use_rx_skb = 0; 127 LKM_MOD_PARAM(use_rx_skb, "i", int, 0); 128 MODULE_PARM_DESC(use_rx_skb, 129 "Use socket buffers for receive operation (default 0)"); 130 131 static int num_rx_prio = 1; 132 LKM_MOD_PARAM(num_rx_prio, "i", int, 0); 133 MODULE_PARM_DESC(num_rx_prio, 134 "Number of filter priorities per Rx DMA channel"); 135 136 static int rx_rate[8] = { 100000, 100000, 100000, 100000, 100000, 100000, 100000, 0 }; 137 LKM_MOD_PARAM_ARRAY(rx_rate, "1-4i", int, NULL, 0); 138 MODULE_PARM_DESC(rx_rate, 139 "Rx rate in packets per second (default 100000)"); 140 141 static int rx_burst[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; 142 LKM_MOD_PARAM_ARRAY(rx_burst, "1-4i", int, NULL, 0); 143 MODULE_PARM_DESC(rx_burst, 144 "Rx rate burst maximum in packets (default rx_rate/10)"); 145 146 static int check_rcpu_signature = 0; 147 LKM_MOD_PARAM(check_rcpu_signature, "i", int, 0); 148 MODULE_PARM_DESC(check_rcpu_signature, 149 "Check RCPU Signature for Tx packets from RCPU interfaces"); 150 151 static int basedev_suspend = 0; 152 LKM_MOD_PARAM(basedev_suspend, "i", int, 0); 153 MODULE_PARM_DESC(basedev_suspend, 154 "Pause traffic till base device is up (enabled by default in NAPI mode)"); 155 156 /* Debug levels */ 157 #define DBG_LVL_VERB 0x1 158 #define DBG_LVL_DCB 0x2 159 #define DBG_LVL_PKT 0x4 160 #define DBG_LVL_SKB 0x8 161 #define DBG_LVL_CMD 0x10 162 #define DBG_LVL_EVT 0x20 163 #define DBG_LVL_IRQ 0x40 164 #define DBG_LVL_NAPI 0x80 165 #define DBG_LVL_PDMP 0x100 166 #define DBG_LVL_FLTR 0x200 167 #define DBG_LVL_KCOM 0x400 168 #define DBG_LVL_RCPU 0x800 169 #define DBG_LVL_WARN 0x1000 170 #define DBG_LVL_NDEV 0x2000 171 #define DBG_LVL_INST 0x4000 172 /* Level to output Dune internal headers Parsing */ 173 #define DBG_LVL_DUNE 0x8000 174 #define DBG_LVL_DCB_TX 0x10000 175 #define DBG_LVL_DCB_RX 0x20000 176 #define DBG_LVL_PDMP_TX 0x40000 177 #define DBG_LVL_PDMP_RX 0x80000 178 179 #define DBG_VERB(_s) do { if (debug & DBG_LVL_VERB) gprintk _s; } while (0) 180 #define DBG_PKT(_s) do { if (debug & DBG_LVL_PKT) gprintk _s; } while (0) 181 #define DBG_SKB(_s) do { if (debug & DBG_LVL_SKB) gprintk _s; } while (0) 182 #define DBG_CMD(_s) do { if (debug & DBG_LVL_CMD) gprintk _s; } while (0) 183 #define DBG_EVT(_s) do { if (debug & DBG_LVL_EVT) gprintk _s; } while (0) 184 #define DBG_IRQ(_s) do { if (debug & DBG_LVL_IRQ) gprintk _s; } while (0) 185 #define DBG_NAPI(_s) do { if (debug & DBG_LVL_NAPI) gprintk _s; } while (0) 186 #define DBG_PDMP(_s) do { if (debug & DBG_LVL_PDMP) gprintk _s; } while (0) 187 #define DBG_FLTR(_s) do { if (debug & DBG_LVL_FLTR) gprintk _s; } while (0) 188 #define DBG_KCOM(_s) do { if (debug & DBG_LVL_KCOM) gprintk _s; } while (0) 189 #define DBG_RCPU(_s) do { if (debug & DBG_LVL_RCPU) gprintk _s; } while (0) 190 #define DBG_WARN(_s) do { if (debug & DBG_LVL_WARN) gprintk _s; } while (0) 191 #define DBG_NDEV(_s) do { if (debug & DBG_LVL_NDEV) gprintk _s; } while (0) 192 #define DBG_INST(_s) do { if (debug & DBG_LVL_INST) gprintk _s; } while (0) 193 #define DBG_DUNE(_s) do { if (debug & DBG_LVL_DUNE) gprintk _s; } while (0) 194 #define DBG_DCB_TX(_s) do { if (debug & (DBG_LVL_DCB|DBG_LVL_DCB_TX)) \ 195 gprintk _s; } while (0) 196 #define DBG_DCB_RX(_s) do { if (debug & (DBG_LVL_DCB|DBG_LVL_DCB_RX)) \ 197 gprintk _s; } while (0) 198 #define DBG_DCB(_s) do { if (debug & (DBG_LVL_DCB|DBG_LVL_DCB_TX| \ 199 DBG_LVL_DCB_RX)) \ 200 gprintk _s; } while (0) 201 202 203 /* This flag is used to indicate if debugging packet function is open or closed */ 204 static int dbg_pkt_enable = 0; 205 206 /* Module Information */ 207 #define MODULE_MAJOR 122 208 #define MODULE_NAME "linux-bcm-knet" 209 210 #ifndef NAPI_SUPPORT 211 #define NAPI_SUPPORT 1 212 #endif 213 214 #if NAPI_SUPPORT 215 216 static int use_napi = 0; 217 LKM_MOD_PARAM(use_napi, "i", int, 0); 218 MODULE_PARM_DESC(use_napi, 219 "Use NAPI interface (default 0)"); 220 221 static int napi_weight = 64; 222 LKM_MOD_PARAM(napi_weight, "i", int, 0); 223 MODULE_PARM_DESC(napi_weight, 224 "Weight of NAPI interfaces (default 64)"); 225 226 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) 227 #define bkn_napi_enable(_dev, _napi) netif_poll_enable(_dev) 228 #define bkn_napi_disable(_dev, _napi) netif_poll_disable(_dev) 229 #define bkn_napi_schedule(_dev, _napi) netif_rx_schedule(_dev) 230 #define bkn_napi_schedule_prep(_dev, _napi) netif_rx_schedule_prep(_dev) 231 #define __bkn_napi_schedule(_dev, _napi) __netif_rx_schedule(_dev) 232 #define bkn_napi_complete(_dev, _napi) netif_rx_complete(_dev) 233 #else 234 #define bkn_napi_enable(_dev, _napi) napi_enable(_napi) 235 #define bkn_napi_disable(_dev, _napi) napi_disable(_napi) 236 #define bkn_napi_schedule(_dev, _napi) napi_schedule(_napi) 237 #define bkn_napi_schedule_prep(_dev, _napi) napi_schedule_prep(_napi) 238 #define __bkn_napi_schedule(_dev, _napi) __napi_schedule(_napi) 239 #define bkn_napi_complete(_dev, _napi) napi_complete(_napi) 240 #endif 241 242 #else 243 244 static int use_napi = 0; 245 static int napi_weight = 0; 246 247 #define bkn_napi_enable(_dev, _napi) 248 #define bkn_napi_disable(_dev, _napi) 249 #define bkn_napi_schedule(_dev, _napi) 250 #define bkn_napi_schedule_prep(_dev, _napi) (0) 251 #define __bkn_napi_schedule(_dev, _napi) 252 #define bkn_napi_complete(_dev, _napi) 253 254 #endif 255 256 /* 257 * If proxy support is compiled in the module will attempt to use 258 * the user/kernel message service provided by the linux-uk-proxy 259 * kernel module, otherwise device IOCTL will be used. 260 */ 261 #ifndef PROXY_SUPPORT 262 #define PROXY_SUPPORT 0 263 #endif 264 265 #if PROXY_SUPPORT 266 267 #include <linux-uk-proxy.h> 268 269 static int use_proxy = 1; 270 LKM_MOD_PARAM(use_proxy, "i", int, 0); 271 MODULE_PARM_DESC(use_proxy, 272 "Use Linux User/Kernel proxy (default 1)"); 273 274 #define PROXY_SERVICE_CREATE(_s,_q,_f) linux_uk_proxy_service_create(_s,_q,_f) 275 #define PROXY_SERVICE_DESTROY(_s) linux_uk_proxy_service_destroy(_s); 276 #define PROXY_SEND(_s,_m,_l) linux_uk_proxy_send(_s,_m,_l) 277 #define PROXY_RECV(_s,_m,_l) linux_uk_proxy_recv(_s,_m,_l) 278 279 #else 280 281 static int use_proxy = 0; 282 283 #define PROXY_SERVICE_CREATE(_s,_q,_f) 284 #define PROXY_SERVICE_DESTROY(_s) 285 #define PROXY_SEND(_s,_m,_l) 286 #define PROXY_RECV(_s,_m,_l) (-1) 287 288 #endif 289 290 /* Compatibility */ 291 292 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4,7,0)) 293 #define NETDEV_UPDATE_TRANS_START_TIME(dev) dev->trans_start = jiffies 294 #else 295 #define NETDEV_UPDATE_TRANS_START_TIME(dev) netif_trans_update(dev) 296 #endif 297 298 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24)) 299 #define skb_copy_to_linear_data(_skb, _pkt, _len) \ 300 eth_copy_and_sum(_skb, _pkt, _len, 0) 301 struct napi_struct { int not_used; }; 302 #define netif_napi_add(_dev, _napi, _poll, _weight) do { \ 303 (_dev)->poll = _poll; \ 304 (_dev)->weight = _weight; \ 305 } while(0) 306 #endif 307 308 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18)) 309 #define SKB_PADTO(_skb,_len) (((_skb = skb_padto(_skb,_len)) == NULL) ? -1 : 0) 310 #else 311 #define SKB_PADTO(_skb,_len) skb_padto(_skb,_len) 312 #endif 313 314 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,12)) 315 #define skb_header_cloned(_skb) \ 316 skb_cloned(_skb) 317 #endif 318 319 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,4,27) 320 static inline void *netdev_priv(struct net_device *dev) 321 { 322 return dev->priv; 323 } 324 #endif /* KERNEL_VERSION(2,4,27) */ 325 326 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,4,23) 327 /* Special check for MontaVista 2.4.20 MIPS */ 328 #if !(defined(MAX_USER_RT_PRIO) && defined(CONFIG_MIPS)) 329 static inline void free_netdev(struct net_device *dev) 330 { 331 kfree(dev); 332 } 333 #endif 334 static inline void netif_poll_disable(struct net_device *dev) 335 { 336 while (test_and_set_bit(__LINK_STATE_RX_SCHED, &dev->state)) { 337 /* No hurry. */ 338 current->state = TASK_INTERRUPTIBLE; 339 schedule_timeout(1); 340 } 341 } 342 static inline void netif_poll_enable(struct net_device *dev) 343 { 344 clear_bit(__LINK_STATE_RX_SCHED, &dev->state); 345 } 346 #endif /* KERNEL_VERSION(2,4,23) */ 347 348 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,4,21) 349 static struct sk_buff *skb_pad(struct sk_buff *skb, int pad) 350 { 351 struct sk_buff *nskb; 352 353 /* If the skbuff is non linear tailroom is always zero.. */ 354 if(skb_tailroom(skb) >= pad) 355 { 356 memset(skb->data+skb->len, 0, pad); 357 return skb; 358 } 359 360 nskb = skb_copy_expand(skb, skb_headroom(skb), skb_tailroom(skb) + pad, GFP_ATOMIC); 361 kfree_skb(skb); 362 if(nskb) 363 memset(nskb->data+nskb->len, 0, pad); 364 return nskb; 365 } 366 static inline struct sk_buff *skb_padto(struct sk_buff *skb, unsigned int len) 367 { 368 unsigned int size = skb->len; 369 if(likely(size >= len)) 370 return skb; 371 return skb_pad(skb, len-size); 372 } 373 #endif /* KERNEL_VERSION(2,4,21) */ 374 375 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,10,0) 376 #define bkn_vlan_hwaccel_put_tag(_skb, _proto, _tci) \ 377 __vlan_hwaccel_put_tag(_skb, _tci) 378 #else 379 #define bkn_vlan_hwaccel_put_tag(_skb, _proto, _tci) \ 380 __vlan_hwaccel_put_tag(_skb, htons(_proto), _tci) 381 #endif 382 383 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,27) 384 #define bkn_dma_mapping_error(d, a) \ 385 dma_mapping_error(a) 386 #define bkn_pci_dma_mapping_error(d, a) \ 387 pci_dma_mapping_error(a) 388 #else 389 #define bkn_dma_mapping_error(d, a) \ 390 dma_mapping_error(d, a) 391 #define bkn_pci_dma_mapping_error(d, a) \ 392 pci_dma_mapping_error(d, a) 393 #endif 394 395 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,30) 396 enum { 397 SKBTX_HW_TSTAMP = 1 << 0, 398 SKBTX_SW_TSTAMP = 1 << 1, 399 SKBTX_IN_PROGRESS = 1 << 2, 400 }; 401 struct skb_shared_hwtstamps { 402 ktime_t hwtstamp; 403 ktime_t syststamp; 404 }; 405 struct bkn_skb_shared_info { 406 uint8_t tx_flags; 407 struct skb_shared_hwtstamps hwtstamps; 408 }; 409 #define bkn_skb_shinfo(_skb) ((struct bkn_skb_shared_info *)(unsigned char *)_skb->end) 410 #define bkn_skb_tx_flags(_skb) bkn_skb_shinfo(_skb)->tx_flags 411 static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb) 412 { 413 return &bkn_skb_shinfo(skb)->hwtstamps; 414 } 415 void skb_tstamp_tx(struct sk_buff *orig_skb, struct skb_shared_hwtstamps *hwtstamps) 416 { 417 } 418 static inline void bkn_skb_tx_timestamp(struct sk_buff *skb) 419 { 420 } 421 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,26) 422 static inline ktime_t ns_to_ktime(u64 ns) 423 { 424 static const ktime_t ktime; 425 return ktime; 426 } 427 #endif 428 #elif LINUX_VERSION_CODE < KERNEL_VERSION(2,6,37) 429 #include <linux/net_tstamp.h> 430 enum { 431 SKBTX_HW_TSTAMP = 1 << 0, 432 SKBTX_SW_TSTAMP = 1 << 1, 433 SKBTX_IN_PROGRESS = 1 << 2, 434 }; 435 #define bkn_skb_tx_flags(_skb) skb_shinfo(_skb)->tx_flags.flags 436 static inline void bkn_skb_tx_timestamp(struct sk_buff *skb) 437 { 438 } 439 #else 440 #include <linux/net_tstamp.h> 441 #define bkn_skb_tx_flags(_skb) skb_shinfo(_skb)->tx_flags 442 static inline void bkn_skb_tx_timestamp(struct sk_buff *skb) 443 { 444 return skb_tx_timestamp(skb); 445 } 446 #endif 447 448 #ifdef LINUX_BDE_DMA_DEVICE_SUPPORT 449 #define DMA_DEV device 450 #define DMA_FROMDEV DMA_FROM_DEVICE 451 #define DMA_TODEV DMA_TO_DEVICE 452 #define DMA_MAP_SINGLE(d,p,s,r) dma_map_single(d,p,s,r) 453 #define DMA_UNMAP_SINGLE(d,a,s,r) dma_unmap_single(d,a,s,r) 454 #define DMA_ALLOC_COHERENT(d,s,h) dma_alloc_coherent(d,s,h,GFP_ATOMIC|GFP_DMA32) 455 #define DMA_FREE_COHERENT(d,s,a,h) dma_free_coherent(d,s,a,h) 456 #define DMA_MAPPING_ERROR(d,a) bkn_dma_mapping_error(d,a) 457 #else 458 #define DMA_DEV pci_dev 459 #define DMA_FROMDEV PCI_DMA_FROMDEVICE 460 #define DMA_TODEV PCI_DMA_TODEVICE 461 #define DMA_MAP_SINGLE(d,p,s,r) pci_map_single(d,p,s,r) 462 #define DMA_UNMAP_SINGLE(d,a,s,r) pci_unmap_single(d,a,s,r) 463 #define DMA_ALLOC_COHERENT(d,s,h) pci_alloc_consistent(d,s,h) 464 #define DMA_FREE_COHERENT(d,s,a,h) pci_free_consistent(d,s,a,h) 465 #define DMA_MAPPING_ERROR(d,a) bkn_pci_dma_mapping_error(d,a) 466 #endif 467 468 /* RCPU operations */ 469 #define RCPU_OPCODE_RX 0x10 470 #define RCPU_OPCODE_TX 0x20 471 472 /* RCPU flags */ 473 #define RCPU_F_MODHDR 0x4 474 475 /* RCPU encapsulation */ 476 #define RCPU_HDR_SIZE 32 477 #define RCPU_TX_META_SIZE 32 478 #define RCPU_TX_ENCAP_SIZE (RCPU_HDR_SIZE + RCPU_TX_META_SIZE) 479 #define RCPU_RX_META_SIZE 64 480 #define RCPU_RX_ENCAP_SIZE (RCPU_HDR_SIZE + RCPU_RX_META_SIZE) 481 482 #define PKT_TX_HDR_SIZE 16 483 484 static volatile int module_initialized; 485 486 static ibde_t *kernel_bde = NULL; 487 488 /* Descriptor info */ 489 typedef struct bkn_desc_info_s { 490 uint32_t *dcb_mem; 491 uint64_t dcb_dma; 492 struct sk_buff *skb; 493 uint64_t skb_dma; 494 uint32_t dma_size; 495 } bkn_desc_info_t; 496 497 /* DCB chain info */ 498 typedef struct bkn_dcb_chain_s { 499 struct list_head list; 500 int dcb_cnt; 501 int dcb_cur; 502 uint32_t *dcb_mem; 503 uint64_t dcb_dma; 504 } bkn_dcb_chain_t; 505 506 #define MAX_TX_DCBS 64 507 #define MAX_RX_DCBS 64 508 509 #define NUM_DMA_CHAN 8 510 #define NUM_RX_CHAN 7 511 #define NUM_CMICX_RX_CHAN 7 512 #define NUM_CMICM_RX_CHAN 3 513 514 #define FCS_SZ 4 515 #define TAG_SZ 4 516 517 /* Device control info */ 518 typedef struct bkn_switch_info_s { 519 struct list_head list; 520 struct list_head ndev_list; /* Associated virtual Ethernet interfaces */ 521 struct net_device **ndevs; /* Indexed array of ndev_list */ 522 int ndev_max; /* Size of indexed array */ 523 struct list_head rxpf_list; /* Associated Rx packet filters */ 524 volatile void *base_addr; /* Base address for PCI register access */ 525 struct DMA_DEV *dma_dev; /* Required for DMA memory control */ 526 struct pci_dev *pdev; /* Required for DMA memory control */ 527 struct net_device *dev; /* Base network device */ 528 struct napi_struct napi; /* New NAPI */ 529 struct timer_list timer; /* Retry/resource timer */ 530 int timer_queued; /* Flag indicating queued timer function */ 531 uint32_t timer_runs; /* Timer function runs (debug only) */ 532 struct timer_list rxtick; /* Rx rate control timer */ 533 uint32_t rxticks_per_sec; /* Rx rate control update frequency */ 534 uint32_t rxtick_jiffies; /* Time between updates (in jiffies) */ 535 uint32_t rxticks; /* Rx rate control debug counter */ 536 uint32_t interrupts; /* Total number of interrupts */ 537 spinlock_t lock; /* Main lock for device */ 538 int dev_no; /* Device number (from BDE) */ 539 int cpu_no; /* Cpu number. 1 for iHost(AXI),0 for others */ 540 int dcb_type; /* DCB type */ 541 int dcb_wsize; /* DCB size (in 32-bit words) */ 542 int pkt_hdr_size; /* Packet header size */ 543 uint32_t ftmh_lb_key_ext_size; /* FTMH LB-Key Extension existence/size */ 544 uint32_t ftmh_stacking_ext_size; /* FTMH Stacking extension existence/size */ 545 uint32_t pph_base_size; /* Size of PPH base */ 546 uint32_t pph_lif_ext_size[8]; /* Size of PPH Lif extension header */ 547 uint8_t udh_enable; /* Indicates UDH existence */ 548 uint32_t udh_length_type[4]; /* Size of UDH header per type */ 549 int rx_chans; /* Number of Rx channels */ 550 uint32_t dma_hi; /* DMA higher address */ 551 uint32_t cmic_type; /* CMIC type (CMICe or CMICm) */ 552 uint32_t irq_mask; /* Active IRQs for DMA control */ 553 uint32_t napi_poll_mode; /* NAPI is in polling mode */ 554 uint32_t napi_not_done; /* NAPI poll did not process all packets */ 555 uint32_t napi_poll_again; /* Used if DCB chain is restarted */ 556 uint32_t tx_yield; /* Tx schedule for Continuous DMA and Non-NAPI mode */ 557 void *dcb_mem; /* Logical pointer to DCB memory */ 558 uint64_t dcb_dma; /* Physical bus address for DCB memory */ 559 int dcb_mem_size; /* Total size of allocated DCB memory */ 560 uint32_t dma_events; /* DMA events pending for BCM API */ 561 uint32_t rcpu_sig; /* RCPU signature */ 562 uint64_t halt_addr[NUM_DMA_CHAN]; /* DMA halt address */ 563 uint32_t cdma_channels; /* Active channels for Continuous DMA mode */ 564 uint32_t poll_channels; /* Channels for polling */ 565 uint32_t inst_id; /* Instance id of this device */ 566 int evt_idx; /* Event queue index for this device*/ 567 int basedev_suspended; /* Base device suspended */ 568 int tx_hwts; /* HW timestamp for Tx */ 569 int rx_hwts; /* HW timestamp for Rx */ 570 struct sk_buff_head tx_ptp_queue; /* Tx PTP skb queue */ 571 struct work_struct tx_ptp_work; /* Tx PTP work */ 572 struct { 573 bkn_desc_info_t desc[MAX_TX_DCBS+1]; 574 int free; /* Number of free Tx DCBs */ 575 int cur; /* Index of current Tx DCB */ 576 int dirty; /* Index of next Tx DCB to complete */ 577 int api_active; /* BCM Tx API is in progress */ 578 int suspends; /* Calls to netif_stop_queue (debug only) */ 579 struct list_head api_dcb_list; /* Tx DCB chains from BCM Tx API */ 580 bkn_dcb_chain_t *api_dcb_chain; /* Current Tx DCB chain */ 581 bkn_dcb_chain_t *api_dcb_chain_end; /* Tx DCB chain end */ 582 uint32_t pkts; /* Tx packet counter */ 583 uint32_t pkts_d_no_skb; /* Tx drop - skb allocation failed */ 584 uint32_t pkts_d_rcpu_encap; /* Tx drop - bad RCPU encapsulation */ 585 uint32_t pkts_d_rcpu_sig; /* Tx drop - bad RCPU signature */ 586 uint32_t pkts_d_rcpu_meta; /* Tx drop - bad RCPU meta data */ 587 uint32_t pkts_d_pad_fail; /* Tx drop - pad to minimum size failed */ 588 uint32_t pkts_d_dma_resrc; /* Tx drop - no DMA resources */ 589 uint32_t pkts_d_callback; /* Tx drop - consumed by call-back */ 590 uint32_t pkts_d_no_link; /* Tx drop - software link down */ 591 uint32_t pkts_d_over_limit; /* Tx drop - length is out of range */ 592 } tx; 593 struct { 594 bkn_desc_info_t desc[MAX_RX_DCBS+1]; 595 int free; /* Number of free Rx DCBs */ 596 int cur; /* Index of current Rx DCB */ 597 int dirty; /* Index of next Rx DCB to complete */ 598 int running; /* Rx DMA is active */ 599 int api_active; /* BCM Rx API is active */ 600 int chain_complete; /* All DCBs in chain processed */ 601 int sync_err; /* Chain done with incomplete DCBs (debug) */ 602 int sync_retry; /* Total retry times for sync error (debug) */ 603 int sync_maxloop; /* Max loop times once in recovering sync (debug) */ 604 int use_rx_skb; /* Use SKBs for DMA */ 605 uint32_t rate_max; /* Rx rate in packets/sec */ 606 uint32_t burst_max; /* Rx burst size in number of packets */ 607 uint32_t tokens; /* Tokens for Rx rate control */ 608 uint32_t rate; /* Current packet rate */ 609 unsigned long tok_jif; /* Jiffies at last token update */ 610 unsigned long rate_jif; /* Jiffies at last rate update */ 611 struct list_head api_dcb_list; /* Rx DCB chains from BCM Rx API */ 612 bkn_dcb_chain_t *api_dcb_chain; /* Current Rx DCB chain */ 613 bkn_dcb_chain_t *api_dcb_chain_end; /* Rx DCB chain end */ 614 uint32_t pkts; /* Rx packet counter */ 615 uint32_t pkts_ref; /* Rx packet count for rate calculation */ 616 uint32_t pkts_f_api; /* Rx packets filtered to API */ 617 uint32_t pkts_f_netif; /* Rx packets filtered to net interface */ 618 uint32_t pkts_m_api; /* Rx packets mirrored to API */ 619 uint32_t pkts_m_netif; /* Rx packets mirrored to net interface */ 620 uint32_t pkts_d_no_skb; /* Rx drop - skb allocation failed */ 621 uint32_t pkts_d_no_match; /* Rx drop - no matching filters */ 622 uint32_t pkts_d_unkn_netif; /* Rx drop - unknown net interface ID */ 623 uint32_t pkts_d_unkn_dest; /* Rx drop - unknown destination type */ 624 uint32_t pkts_d_callback; /* Rx drop - consumed by call-back */ 625 uint32_t pkts_d_no_link; /* Rx drop - software link down */ 626 uint32_t pkts_d_no_api_buf; /* Rx drop - no API buffers */ 627 } rx[NUM_RX_CHAN]; 628 } bkn_switch_info_t; 629 630 /* PTCH_2 */ 631 #define BKN_DNX_PTCH_2_SIZE 2 632 /* ITMH */ 633 #define BKN_DNX_ITMH_SIZE 5 634 /* Modlue Header */ 635 #define BKN_DNX_MODULE_HEADER_SIZE 20 636 /* FTMH */ 637 #define BKN_DNX_FTMH_SRC_SYS_PORT_AGGREGATE_MSB 17 638 #define BKN_DNX_FTMH_SRC_SYS_PORT_AGGREGATE_NOF_BITS 16 639 #define BKN_DNX_FTMH_PP_DSP_MSB 33 640 #define BKN_DNX_FTMH_PP_DSP_NOF_BITS 8 641 #define BKN_DNX_FTMH_ACTION_TYPE_MSB 43 642 #define BKN_DNX_FTMH_ACTION_TYPE_NOF_BITS 2 643 #define BKN_DNX_FTMH_PPH_TYPE_IS_TSH_EN_MSB 73 644 #define BKN_DNX_FTMH_PPH_TYPE_IS_TSH_EN_NOF_BITS 1 645 #define BKN_DNX_FTMH_PPH_TYPE_IS_PPH_EN_MSB 74 646 #define BKN_DNX_FTMH_PPH_TYPE_IS_PPH_EN_NOF_BITS 1 647 #define BKN_DNX_FTMH_TM_DST_EXT_PRESENT_MSB 75 648 #define BKN_DNX_FTMH_TM_DST_EXT_PRESENT_NOF_BITS 1 649 #define BKN_DNX_FTMH_APP_SPECIFIC_EXT_SIZE_MSB 76 650 #define BKN_DNX_FTMH_APP_SPECIFIC_EXT_SIZE_NOF_BITS 1 651 #define BKN_DNX_FTMH_FLOW_ID_EXT_SIZE_MSB 77 652 #define BKN_DNX_FTMH_FLOW_ID_EXT_SIZE_NOF_BITS 1 653 #define BKN_DNX_FTMH_BIER_BFR_EXT_SIZE_MSB 78 654 #define BKN_DNX_FTMH_BIER_BFR_EXT_SIZE_NOF_BITS 1 655 /* Fix Length for FTMH and Extension headers */ 656 #define BKN_DNX_FTMH_BASE_SIZE 10 657 #define BKN_DNX_FTMH_BIER_BFR_EXT_SIZE 2 658 #define BKN_DNX_FTMH_TM_DST_EXT_SIZE 3 659 #define BKN_DNX_FTMH_FLOW_ID_EXT_SIZE 3 660 #define BKN_DNX_FTMH_APP_SPECIFIC_EXT_SIZE 6 661 /* TSH */ 662 #define BKN_DNX_TSH_SIZE 4 663 /* PPH */ 664 #define BKN_DNX_PPH_BASE_TYPE_9 9 665 #define BKN_DNX_PPH_BASE_TYPE_10 10 666 #define BKN_DNX_PPH_BASE_TYPE_12 12 667 #define BKN_DNX_PPH_9_FORWARD_DOMAIN_MSB 5 668 #define BKN_DNX_PPH_9_FORWARD_DOMAIN_NOF_BITS 16 669 #define BKN_DNX_PPH_9_LEARN_EXT_PRESENT_MSB 53 670 #define BKN_DNX_PPH_9_LEARN_EXT_PRESENT_NOF_BITS 1 671 #define BKN_DNX_PPH_9_FHEI_SIZE_MSB 54 672 #define BKN_DNX_PPH_9_FHEI_SIZE_NOF_BITS 2 673 #define BKN_DNX_PPH_9_LIF_EXT_TYPE_MSB 56 674 #define BKN_DNX_PPH_9_LIF_EXT_TYPE_NOF_BITS 3 675 #define BKN_DNX_PPH_10_FORWARD_DOMAIN_MSB 9 676 #define BKN_DNX_PPH_10_FORWARD_DOMAIN_NOF_BITS 16 677 #define BKN_DNX_PPH_10_LEARN_EXT_PRESENT_MSB 61 678 #define BKN_DNX_PPH_10_LEARN_EXT_PRESENT_NOF_BITS 1 679 #define BKN_DNX_PPH_10_FHEI_SIZE_MSB 62 680 #define BKN_DNX_PPH_10_FHEI_SIZE_NOF_BITS 2 681 #define BKN_DNX_PPH_10_LIF_EXT_TYPE_MSB 64 682 #define BKN_DNX_PPH_10_LIF_EXT_TYPE_NOF_BITS 3 683 #define BKN_DNX_PPH_12_FORWARD_DOMAIN_MSB 21 684 #define BKN_DNX_PPH_12_FORWARD_DOMAIN_NOF_BITS 18 685 #define BKN_DNX_PPH_12_LEARN_EXT_PRESENT_MSB 77 686 #define BKN_DNX_PPH_12_LEARN_EXT_PRESENT_NOF_BITS 1 687 #define BKN_DNX_PPH_12_FHEI_SIZE_MSB 78 688 #define BKN_DNX_PPH_12_FHEI_SIZE_NOF_BITS 2 689 #define BKN_DNX_PPH_12_LIF_EXT_TYPE_MSB 80 690 #define BKN_DNX_PPH_12_LIF_EXT_TYPE_NOF_BITS 3 691 /* PPH.FHEI_TYPE */ 692 #define BKN_DNX_PPH_FHEI_TYPE_SZ0 1 693 #define BKN_DNX_PPH_FHEI_TYPE_SZ1 2 694 #define BKN_DNX_PPH_FHEI_TYPE_SZ2 3 695 /* FHEI */ 696 #define BKN_DNX_PPH_FHEI_SZ0_SIZE 3 697 #define BKN_DNX_PPH_FHEI_SZ1_SIZE 5 698 #define BKN_DNX_PPH_FHEI_SZ2_SIZE 8 699 #define BKN_DNX_PPH_FHEI_TRAP_5B_QUALIFIER_MSB 0 700 #define BKN_DNX_PPH_FHEI_TRAP_5B_QUALIFIER_NOF_BITS 27 701 #define BKN_DNX_PPH_FHEI_TRAP_5B_CODE_MSB 27 702 #define BKN_DNX_PPH_FHEI_TRAP_5B_CODE_NOF_BITS 9 703 #define BKN_DNX_PPH_FHEI_TRAP_5B_TYPE_MSB 36 704 #define BKN_DNX_PPH_FHEI_TRAP_5B_TYPE_NOF_BITS 4 705 /* PPH Extension */ 706 #define BKN_DNX_PPH_LEARN_EXT_SIZE 19 707 /* UDH */ 708 #define BKN_DNX_UDH_DATA_TYPE_0_MSB 0 709 #define BKN_DNX_UDH_DATA_TYPE_0_NOF_BITS 2 710 #define BKN_DNX_UDH_DATA_TYPE_1_MSB 2 711 #define BKN_DNX_UDH_DATA_TYPE_1_NOF_BITS 2 712 #define BKN_DNX_UDH_DATA_TYPE_2_MSB 4 713 #define BKN_DNX_UDH_DATA_TYPE_2_NOF_BITS 2 714 #define BKN_DNX_UDH_DATA_TYPE_3_MSB 6 715 #define BKN_DNX_UDH_DATA_TYPE_3_NOF_BITS 2 716 #define BKN_DNX_UDH_BASE_SIZE 1 717 718 #define BKN_DPP_HDR_MAX_SIZE 40 719 /* PTCH_2 */ 720 #define BKN_DPP_PTCH_2_SIZE 2 721 /* ITMH */ 722 #define BKN_DPP_ITMH_SIZE 4 723 /* FTMH */ 724 #define BKN_DPP_FTMH_LB_EXT_EN 0x1 725 #define BKN_DPP_FTMH_STACKING_EXT_EN 0x2 726 #define BKN_DPP_FTMH_SIZE_BYTE 9 727 #define BKN_DPP_FTMH_LB_EXT_SIZE_BYTE 1 728 #define BKN_DPP_FTMH_STACKING_SIZE_BYTE 2 729 #define BKN_DPP_FTMH_DEST_EXT_SIZE_BYTE 2 730 #define BKN_DPP_FTMH_LB_EXT_SIZE_BYTE 1 731 #define BKN_DPP_FTMH_PKT_SIZE_MSB 0 732 #define BKN_DPP_FTMH_PKT_SIZE_NOF_BITS 14 733 #define BKN_DPP_FTMH_TC_MSB 14 734 #define BKN_DPP_FTMH_TC_NOF_BITS 3 735 #define BKN_DPP_FTMH_SRC_SYS_PORT_MSB 17 736 #define BKN_DPP_FTMH_SRC_SYS_PORT_NOF_BITS 16 737 #define BKN_DPP_FTMH_EXT_DSP_EXIST_MSB 68 738 #define BKN_DPP_FTMH_EXT_DSP_EXIST_NOF_BITS 1 739 #define BKN_DPP_FTMH_EXT_MSB 45 740 #define BKN_DPP_FTMH_EXT_NOF_BITS 2 741 #define BKN_DPP_FTMH_FIRST_EXT_MSB 72 742 #define BKN_DPP_FTMH_ACTION_TYPE_MSB 43 743 #define BKN_DPP_FTMH_ACTION_TYPE_NOF_BITS 2 744 #define BKN_DPP_FTMH_PPH_TYPE_MSB 45 745 #define BKN_DPP_FTMH_PPH_TYPE_NOF_BITS 2 746 747 /* PPH */ 748 #define BKN_DPP_PPH_SIZE_BYTE 7 749 #define BKN_DPP_PPH_EEI_EXTENSION_PRESENT_MSB 0 750 #define BKN_DPP_PPH_EEI_EXTENSION_PRESENT_NOF_BITS 1 751 #define BKN_DPP_PPH_LEARN_EXENSION_PRESENT_MSB 1 752 #define BKN_DPP_PPH_LEARN_EXENSION_PRESENT_NOF_BITS 1 753 #define BKN_DPP_PPH_FHEI_SIZE_MSB 2 754 #define BKN_DPP_PPH_FHEI_SIZE_NOF_BITS 2 755 #define BKN_DPP_PPH_FORWARD_CODE_MSB 4 756 #define BKN_DPP_PPH_FORWARD_CODE_NOF_BITS 4 757 #define BKN_DPP_PPH_VSI_MSB 22 758 #define BKN_DPP_PPH_VSI_NOF_BITS 16 759 /* FHEI TRAP/SNOOP 3B */ 760 #define BKN_DPP_PPH_FHEI_3B_SIZE_BYTE 3 761 #define BKN_DPP_PPH_FHEI_5B_SIZE_BYTE 5 762 #define BKN_DPP_PPH_FHEI_8B_SIZE_BYTE 8 763 #define BKN_DPP_PPH_FHEI_TRAP_SNOOP_3B_CPU_TRAP_CODE_QUALIFIER_MSB 0 764 #define BKN_DPP_PPH_FHEI_TRAP_SNOOP_3B_CPU_TRAP_CODE_QUALIFIER_NOF_BITS 16 765 #define BKN_DPP_PPH_FHEI_TRAP_SNOOP_3B_CPU_TRAP_CODE_MSB 16 766 #define BKN_DPP_PPH_FHEI_TRAP_SNOOP_3B_CPU_TRAP_CODE_NOF_BITS 8 767 /* PPH extension */ 768 #define BKN_DPP_PPH_EXPLICIT_EDITING_INFOMATION_EXTENSION_SIZE_BYTE 3 769 #define BKN_DPP_PPH_LEARN_EXTENSION_SIZE_BYTE 5 770 771 /* ftmh action type. */ 772 typedef enum bkn_dpp_ftmh_action_type_e { 773 BKN_DPP_FTMH_ACTION_TYPE_FORWARD = 0, /* TM action is forward */ 774 BKN_DPP_FTMH_ACTION_TYPE_SNOOP = 1, /* TM action is snoop */ 775 BKN_DPP_FTMH_ACTION_TYPE_INBOUND_MIRROR = 2, /* TM action is inbound mirror. */ 776 BKN_DPP_FTMH_ACTION_TYPE_OUTBOUND_MIRROR = 3 /* TM action is outbound mirror. */ 777 }bkn_dpp_ftmh_action_type_t; 778 779 /* ftmh dest extension. */ 780 typedef struct bkn_dpp_ftmh_dest_extension_s { 781 uint8 valid; /* Set if the extension is present */ 782 uint32_t dst_sys_port; /* Destination System Port */ 783 } bkn_dpp_ftmh_dest_extension_t; 784 785 /* dnx packet */ 786 typedef struct bkn_dune_system_header_info_s { 787 uint32_t ntwrk_header_ptr; 788 struct { 789 uint32_t packet_size; /* Packet size in bytes */ 790 uint32_t action_type; /* Indicates if the copy is one of the Forward Snoop or Mirror packet copies */ 791 uint32_t pph_type; 792 uint32_t prio; /* Traffic class */ 793 uint32_t src_sys_port; /* Source System port*/ 794 } ftmh; 795 struct { 796 uint32_t vsi; 797 uint32_t trap_qualifier; /* RAW Data */ 798 uint32_t trap_id; /* RAW Data */ 799 } internal; 800 uint32_t system_header_size; 801 uint32_t ftmh_spa; /* FTMH: Source-System-Port-Aggregate*/ 802 uint32_t pph_forward_domain; /* PPH: Forward-Domain*/ 803 uint32_t fhei_qualifier; /* FHEI: Qualifier */ 804 uint32_t fhei_code; /* FHEI: Code */ 805 uint32_t fhei_type; /* FHEI: Type */ 806 } bkn_dune_system_header_info_t; 807 808 #define PREV_IDX(_cur, _max) (((_cur) == 0) ? (_max) - 1 : (_cur) - 1) 809 810 #if defined(CMIC_SOFT_BYTE_SWAP) 811 812 #define CMIC_SWAP32(_x) ((((_x) & 0xff000000) >> 24) \ 813 | (((_x) & 0x00ff0000) >> 8) \ 814 | (((_x) & 0x0000ff00) << 8) \ 815 | (((_x) & 0x000000ff) << 24)) 816 817 #define DEV_READ32(_d, _a, _p) \ 818 do { \ 819 uint32_t _data; \ 820 _data = (((volatile uint32_t *)(_d)->base_addr)[(_a)/4]); \ 821 *(_p) = CMIC_SWAP32(_data); \ 822 } while(0) 823 824 #define DEV_WRITE32(_d, _a, _v) \ 825 do { \ 826 uint32_t _data = CMIC_SWAP32(_v); \ 827 ((volatile uint32_t *)(_d)->base_addr)[(_a)/4] = (_data); \ 828 } while(0) 829 830 #else 831 832 #define DEV_READ32(_d, _a, _p) \ 833 do { \ 834 *(_p) = (((volatile uint32_t *)(_d)->base_addr)[(_a)/4]); \ 835 } while(0) 836 837 #define DEV_WRITE32(_d, _a, _v) \ 838 do { \ 839 ((volatile uint32_t *)(_d)->base_addr)[(_a)/4] = (_v); \ 840 } while(0) 841 842 #endif /* defined(CMIC_SOFT_BYTE_SWAP) */ 843 844 #define MEMORY_BARRIER smp_mb() 845 846 /* Default random MAC address has Broadcom OUI with local admin bit set */ 847 static u8 bkn_dev_mac[6] = { 0x02, 0x10, 0x18, 0x00, 0x00, 0x00 }; 848 849 static u8 bkn_rcpu_dmac[6]; 850 static u8 bkn_rcpu_smac[6]; 851 852 /* Driver Proc Entry root */ 853 static struct proc_dir_entry *bkn_proc_root = NULL; 854 855 typedef struct bkn_priv_s { 856 struct list_head list; 857 struct net_device_stats stats; 858 struct net_device *dev; 859 bkn_switch_info_t *sinfo; 860 int id; 861 int type; 862 int port; 863 uint8_t itmh[4]; 864 int qnum; 865 uint32_t vlan; 866 uint32_t flags; 867 uint32_t cb_user_data; 868 uint8_t system_headers[27]; 869 uint32_t system_headers_size; 870 } bkn_priv_t; 871 872 typedef struct bkn_filter_s { 873 struct list_head list; 874 int dev_no; 875 unsigned long hits; 876 kcom_filter_t kf; 877 } bkn_filter_t; 878 879 880 /* 881 * Multiple instance support in KNET 882 */ 883 static int _bkn_multi_inst = 0; 884 typedef struct { 885 wait_queue_head_t evt_wq; 886 int evt_wq_put; 887 int evt_wq_get; 888 uint32_t inst_id; 889 } bkn_evt_resource_t; 890 891 static bkn_evt_resource_t _bkn_evt[LINUX_BDE_MAX_DEVICES]; 892 893 static int bkn_knet_dev_init(int d); 894 static int bkn_knet_dev_reinit(int d); 895 896 /* IOCTL debug counters */ 897 static int ioctl_cmd; 898 static int ioctl_evt; 899 900 /* Switch devices */ 901 LIST_HEAD(_sinfo_list); 902 903 /* Reallocation chunk size for netif array */ 904 #define NDEVS_CHUNK 64 905 906 /* User call-backs */ 907 static knet_skb_cb_f knet_rx_cb = NULL; 908 static knet_skb_cb_f knet_tx_cb = NULL; 909 static knet_filter_cb_f knet_filter_cb = NULL; 910 static knet_hw_tstamp_enable_cb_f knet_hw_tstamp_enable_cb = NULL; 911 static knet_hw_tstamp_enable_cb_f knet_hw_tstamp_disable_cb = NULL; 912 static knet_hw_tstamp_tx_time_get_cb_f knet_hw_tstamp_tx_time_get_cb = NULL; 913 static knet_hw_tstamp_tx_meta_get_cb_f knet_hw_tstamp_tx_meta_get_cb = NULL; 914 static knet_hw_tstamp_ptp_clock_index_cb_f knet_hw_tstamp_ptp_clock_index_cb = NULL; 915 static knet_hw_tstamp_rx_time_upscale_cb_f knet_hw_tstamp_rx_time_upscale_cb = NULL; 916 917 /* 918 * Thread management 919 */ 920 921 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,10)) 922 /* 923 * Old style using kernel_thread() 924 */ 925 typedef struct { 926 const char * name; 927 volatile int pid; 928 volatile int run; 929 struct completion completion; 930 int state; 931 } bkn_thread_ctrl_t; 932 933 static int 934 bkn_thread_start(bkn_thread_ctrl_t *tc, const char *name, 935 int (*threadfn)(void *)) 936 { 937 if (name == NULL) { 938 return -1; 939 } 940 tc->name = name; 941 tc->pid = kernel_thread(threadfn, tc, 0); 942 if (tc->pid < 0) { 943 tc->pid = 0; 944 return -1; 945 } 946 tc->run = 1; 947 init_completion(&tc->completion); 948 return 0; 949 } 950 951 static int 952 bkn_thread_stop(bkn_thread_ctrl_t *tc) 953 { 954 if (tc->pid == 0) { 955 return 0; 956 } 957 tc->run = 0; 958 kill_proc(tc->pid, SIGTERM, 1); 959 wait_for_completion(&tc->completion); 960 return 0; 961 } 962 963 static int 964 bkn_thread_should_stop(bkn_thread_ctrl_t *tc) 965 { 966 if (tc->run) { 967 return 0; 968 } 969 tc->pid = 0; 970 return 1; 971 } 972 973 static void 974 bkn_thread_boot(bkn_thread_ctrl_t *tc) 975 { 976 siginitsetinv(¤t->blocked, sigmask(SIGTERM) | sigmask(SIGKILL)); 977 } 978 979 static void 980 bkn_thread_exit(bkn_thread_ctrl_t *tc) 981 { 982 complete_and_exit(&tc->completion, 0); 983 } 984 985 static void 986 bkn_sleep(int clicks) 987 { 988 wait_queue_head_t wq; 989 990 init_waitqueue_head(&wq); 991 sleep_on_timeout(&wq, clicks); 992 } 993 #else 994 /* 995 * New style using kthread API 996 */ 997 #include <linux/kthread.h> 998 typedef struct { 999 const char * name; 1000 struct task_struct *task; 1001 int state; 1002 } bkn_thread_ctrl_t; 1003 1004 static int 1005 bkn_thread_start(bkn_thread_ctrl_t *tc, const char *name, 1006 int (*threadfn)(void *)) 1007 { 1008 if (name == NULL) { 1009 return -1; 1010 } 1011 tc->name = name; 1012 tc->task = kthread_run(threadfn, tc, name); 1013 if (IS_ERR(tc->task)) { 1014 tc->task = NULL; 1015 return -1; 1016 } 1017 return 0; 1018 } 1019 1020 static int 1021 bkn_thread_stop(bkn_thread_ctrl_t *tc) 1022 { 1023 if (tc->task == NULL) { 1024 return 0; 1025 } 1026 send_sig(SIGTERM, tc->task, 0); 1027 return kthread_stop(tc->task); 1028 } 1029 1030 static int 1031 bkn_thread_should_stop(bkn_thread_ctrl_t *tc) 1032 { 1033 return kthread_should_stop(); 1034 } 1035 1036 static void 1037 bkn_thread_boot(bkn_thread_ctrl_t *tc) 1038 { 1039 allow_signal(SIGTERM); 1040 allow_signal(SIGKILL); 1041 } 1042 1043 static void 1044 bkn_thread_exit(bkn_thread_ctrl_t *tc) 1045 { 1046 } 1047 1048 static void 1049 bkn_sleep(int clicks) 1050 { 1051 wait_queue_head_t wq; 1052 1053 init_waitqueue_head(&wq); 1054 wait_event_timeout(wq, 0, clicks); 1055 } 1056 #endif 1057 1058 static bkn_thread_ctrl_t bkn_cmd_ctrl; 1059 static bkn_thread_ctrl_t bkn_evt_ctrl; 1060 1061 /* 1062 * On XGS devices bit 15 fo the Transferred Bytes field in 1063 * the DCBs is used to indicate that kernel processing is 1064 * complete. Using this bit reduces the theoretically maximum 1065 * supported packet size from 64K to 32K, but this is still 1066 * adequate for 16K jumbo packets. 1067 */ 1068 #define SOC_DCB_KNET_DONE 0x8000 1069 #define SOC_DCB_KNET_COUNT_MASK 0x7fff 1070 #define SOC_DCB_META_OFFSET 2 1071 1072 /* Default channel configuration */ 1073 #define XGS_DMA_TX_CHAN 0 1074 #define XGS_DMA_RX_CHAN 1 1075 1076 /* CMIC registers */ 1077 #define CMIC_DMA_CTRLr 0x00000100 1078 #define CMIC_DMA_STATr 0x00000104 1079 #define CMIC_DMA_DESC0r 0x00000110 1080 #define CMIC_IRQ_STATr 0x00000144 1081 #define CMIC_IRQ_MASKr 0x00000148 1082 #define CMIC_DEV_REV_IDr 0x00000178 1083 1084 /* CMIC interrupts reserved for kernel handler */ 1085 #define CMIC_TXRX_IRQ_MASK 0x7f80 1086 1087 /* CMICm registers */ 1088 #define CMICM_CMC_BASE 0x00031000 1089 #define CMICM_DMA_CTRLr (CMICM_CMC_BASE + 0x00000140) 1090 #define CMICM_DMA_STATr (CMICM_CMC_BASE + 0x00000150) 1091 #define CMICM_DMA_STAT_CLRr (CMICM_CMC_BASE + 0x000001a4) 1092 #define CMICM_DMA_DESC0r (CMICM_CMC_BASE + 0x00000158) 1093 #define CMICM_DMA_HALT_ADDRr (CMICM_CMC_BASE + 0x00000120) 1094 #define CMICM_IRQ_STATr (CMICM_CMC_BASE + 0x00000400) 1095 #define CMICM_IRQ_PCI_MASKr (CMICM_CMC_BASE + 0x00000414) 1096 #define CMICM_IRQ_UC0_MASKr (CMICM_CMC_BASE + 0x00000428) 1097 #define CMICM_DEV_REV_IDr 0x00010224 1098 1099 /* CMICm interrupts reserved for kernel handler */ 1100 #define CMICM_TXRX_IRQ_MASK 0xff00 1101 1102 /* CMICd increased interrupts reserved for kernel handler */ 1103 #define CMICD_CTRLD_IRQ_MASK 0x78000000 1104 1105 /* CMICx registers */ 1106 #define CMICX_CMC_BASE 0x00000000 1107 #define CMICX_DMA_CTRLr (CMICX_CMC_BASE + 0x00002100) 1108 #define CMICX_DMA_STATr (CMICX_CMC_BASE + 0x00002114) 1109 #define CMICX_DMA_DESC_HIr (CMICX_CMC_BASE + 0x00002108) 1110 #define CMICX_DMA_DESC_LOr (CMICX_CMC_BASE + 0x00002104) 1111 #define CMICX_DMA_HALT_HIr (CMICX_CMC_BASE + 0x00002110) 1112 #define CMICX_DMA_HALT_LOr (CMICX_CMC_BASE + 0x0000210c) 1113 #define CMICX_IRQ_STATr (CMICX_CMC_BASE + 0x0000106c) 1114 #define CMICX_IRQ_STAT_CLRr (CMICX_CMC_BASE + 0x00001074) 1115 #define CMICX_IRQ_ENABr 0x18013100 1116 #define IHOST_GIC_GIC400_GICD_ISENABLERN_5r 0x10781114 1117 #define IHOST_GIC_GIC400_GICD_ICENABLERN_5r 0x10781194 1118 1119 /* CMICx interrupts reserved for kernel handler */ 1120 #define CMICX_TXRX_IRQ_MASK 0xffffffff 1121 1122 #define DEV_IS_CMICX(_sinfo) ((_sinfo)->cmic_type == 'x') 1123 #define DEV_IS_CMICM(_sinfo) ((_sinfo)->cmic_type == 'm') 1124 #define CDMA_CH(_d, _ch) ((_d)->cdma_channels & (1 << (_ch))) 1125 1126 /* 1127 * DMA_STAT: control bits 1128 * 1129 * xxx_SET and xxx_CLR can be WRITTEN to CMIC_DMA_STAT 1130 * xxx_TST can be masked against values read from CMIC_DMA_STAT. 1131 * Argument required: 0 <= ch <= 3 1132 */ 1133 #define DS_DMA_ACTIVE(ch) (0x00040000 << (ch)) 1134 #define DS_DMA_EN_SET(ch) (0x80|(ch)) 1135 #define DS_DMA_EN_CLR(ch) (0x00|(ch)) 1136 #define DS_DMA_EN_TST(ch) (0x00000001 << (ch)) 1137 1138 #define DS_CHAIN_DONE_SET(ch) (0x80|(4+(ch))) 1139 #define DS_CHAIN_DONE_CLR(ch) (0x00|(4+(ch))) 1140 #define DS_CHAIN_DONE_TST(ch) (0x00000010 << (ch)) 1141 1142 #define DS_DESC_DONE_SET(ch) (0x80|(8+(ch))) 1143 #define DS_DESC_DONE_CLR(ch) (0x00|(8+(ch))) 1144 #define DS_DESC_DONE_TST(ch) (0x00000100 << (ch)) 1145 1146 #define DC_ABORT_DMA(ch) (0x04 << (8 * (ch))) 1147 1148 #define DS_CMC_DESCRD_CMPLT_CLR(ch) (0x00000001 << (ch)) 1149 #define DS_CMC_CTRLD_INT_CLR(ch) (0x00000100 << (ch)) 1150 #define DS_CMC_DMA_ACTIVE(ch) (0x00000100 << (ch)) 1151 1152 #define CMICX_DS_CMC_DESC_DONE(ch) (0x00000001 << ((ch) * 4)) 1153 #define CMICX_DS_CMC_CHAIN_DONE(ch) (0x00000002 << ((ch) * 4)) 1154 #define CMICX_DS_CMC_CTRLD_INT(ch) (0x00000008 << ((ch) * 4)) 1155 #define CMICX_DS_CMC_DMA_CHAIN_DONE (0x00000001) 1156 #define CMICX_DS_CMC_DMA_ACTIVE (0x00000002) 1157 1158 #define DMA_TO_BUS_HI(dma) ((dma) | sinfo->dma_hi) 1159 #define BUS_TO_DMA_HI(bus) ((bus) & ~sinfo->dma_hi) 1160 1161 /* 1162 * DMA_CTRL: control bits 1163 */ 1164 #define DC_CMC_DIRECTION (0x00000001) 1165 #define DC_CMC_ENABLE (0x00000002) 1166 #define DC_CMC_ABORT (0x00000004) 1167 #define DC_CMC_CTRLD_INT (0x00000100) 1168 #define DC_CMC_CONTINUOUS (0x00000200) 1169 1170 #define CMICX_DC_CMC_DIRECTION (0x00000001) 1171 #define CMICX_DC_CMC_ENABLE (0x00000002) 1172 #define CMICX_DC_CMC_ABORT (0x00000004) 1173 #define CMICX_DC_CMC_CTRLD_INT (0x00000080) 1174 #define CMICX_DC_CMC_CONTINUOUS (0x00000100) 1175 1176 /* 1177 * Per-channel operations. 1178 * These are the basis for the TX/RX functions 1179 */ 1180 1181 static inline void 1182 xgs_dma_chain_clear(bkn_switch_info_t *sinfo, int chan) 1183 { 1184 DBG_IRQ(("Clear chain on device %d chan %d\n", 1185 sinfo->dev_no, chan)); 1186 DEV_WRITE32(sinfo, CMIC_DMA_STATr, DS_DMA_EN_CLR(chan)); 1187 DEV_WRITE32(sinfo, CMIC_DMA_STATr, DS_CHAIN_DONE_CLR(chan)); 1188 1189 MEMORY_BARRIER; 1190 } 1191 1192 static inline void 1193 xgs_dma_desc_clear(bkn_switch_info_t *sinfo, int chan) 1194 { 1195 uint32_t val; 1196 1197 DBG_IRQ(("Clear desc on device %d chan %d\n", 1198 sinfo->dev_no, chan)); 1199 DEV_WRITE32(sinfo, CMIC_DMA_STATr, DS_DESC_DONE_CLR(chan)); 1200 1201 MEMORY_BARRIER; 1202 1203 /* Flush write buffer */ 1204 DEV_READ32(sinfo, CMIC_DMA_STATr, &val); 1205 1206 MEMORY_BARRIER; 1207 } 1208 1209 static int 1210 xgs_dma_chan_clear(bkn_switch_info_t *sinfo, int chan) 1211 { 1212 xgs_dma_chain_clear(sinfo, chan); 1213 xgs_dma_desc_clear(sinfo, chan); 1214 1215 return 0; 1216 } 1217 1218 static int 1219 xgs_dma_chan_init(bkn_switch_info_t *sinfo, int chan, int dir) 1220 { 1221 uint32_t cdc; 1222 1223 DEV_READ32(sinfo, CMIC_DMA_CTRLr, &cdc); 1224 1225 cdc &= ~(0x9 << (8 * chan)); 1226 if (dir) { 1227 cdc |= 0x1 << (8 * chan); 1228 } else { 1229 cdc |= 0x8 << (8 * chan); 1230 } 1231 1232 DEV_WRITE32(sinfo, CMIC_DMA_CTRLr, cdc); 1233 1234 return 0; 1235 } 1236 1237 static int 1238 xgs_dma_chan_start(bkn_switch_info_t *sinfo, int chan, uint64_t dcb) 1239 { 1240 /* Write the DCB address to the DESC address for this channel */ 1241 DEV_WRITE32(sinfo, CMIC_DMA_DESC0r + 4 * chan, dcb); 1242 1243 MEMORY_BARRIER; 1244 1245 /* Kick it off */ 1246 DEV_WRITE32(sinfo, CMIC_DMA_STATr, DS_DMA_EN_SET(chan)); 1247 1248 MEMORY_BARRIER; 1249 1250 return 0; 1251 } 1252 1253 static int 1254 xgs_dma_chan_abort(bkn_switch_info_t *sinfo, int chan, int polls) 1255 { 1256 uint32_t ctrl, dma_stat; 1257 int p; 1258 1259 /* Clear enable */ 1260 DEV_WRITE32(sinfo, CMIC_DMA_STATr, DS_DMA_EN_CLR(chan)); 1261 1262 MEMORY_BARRIER; 1263 1264 /* Abort the channel */ 1265 DEV_READ32(sinfo, CMIC_DMA_CTRLr, &ctrl); 1266 DEV_WRITE32(sinfo, CMIC_DMA_CTRLr, ctrl | DC_ABORT_DMA(chan)); 1267 1268 MEMORY_BARRIER; 1269 1270 /* Poll for abort completion */ 1271 for (p = 0; p < polls; p++) { 1272 DEV_READ32(sinfo, CMIC_DMA_STATr, &dma_stat); 1273 if (!(dma_stat & DS_DMA_ACTIVE(chan))) { 1274 /* Restore previous control value */ 1275 DEV_WRITE32(sinfo, CMIC_DMA_CTRLr, ctrl); 1276 1277 MEMORY_BARRIER; 1278 1279 /* Clear up channel */ 1280 xgs_dma_chan_clear(sinfo, chan); 1281 1282 return polls; 1283 } 1284 } 1285 DBG_WARN(("DMA channel %d abort failed\n", chan)); 1286 1287 return -1; 1288 } 1289 1290 static inline void 1291 xgs_irq_mask_set(bkn_switch_info_t *sinfo, uint32_t mask) 1292 { 1293 if (sinfo->napi_poll_mode) { 1294 mask = 0; 1295 } 1296 lkbde_irq_mask_set(sinfo->dev_no | LKBDE_ISR2_DEV, CMIC_IRQ_MASKr, 1297 mask, CMIC_TXRX_IRQ_MASK); 1298 } 1299 1300 static inline void 1301 xgs_irq_mask_enable(bkn_switch_info_t *sinfo, int chan, int update_hw) 1302 { 1303 uint32_t mask; 1304 1305 if (chan == XGS_DMA_TX_CHAN) { 1306 mask = 0x100; 1307 } else { 1308 mask = 0x180 << (2 * chan); 1309 } 1310 1311 sinfo->irq_mask |= mask; 1312 1313 if (update_hw) { 1314 xgs_irq_mask_set(sinfo, sinfo->irq_mask); 1315 } 1316 } 1317 1318 static inline void 1319 xgs_irq_mask_disable(bkn_switch_info_t *sinfo, int chan, int update_hw) 1320 { 1321 uint32_t mask; 1322 1323 if (chan == XGS_DMA_TX_CHAN) { 1324 mask = 0x100; 1325 } else { 1326 mask = 0x180 << (2 * chan); 1327 } 1328 1329 sinfo->irq_mask &= ~mask; 1330 1331 if (update_hw) { 1332 xgs_irq_mask_set(sinfo, sinfo->irq_mask); 1333 } 1334 } 1335 1336 static inline void 1337 xgsm_dma_chain_clear(bkn_switch_info_t *sinfo, int chan) 1338 { 1339 uint32_t cdc; 1340 1341 /* Disabing DMA clears chain done */ 1342 DEV_READ32(sinfo, CMICM_DMA_CTRLr + 4 * chan, &cdc); 1343 cdc &= ~(DC_CMC_ENABLE | DC_CMC_ABORT); 1344 DEV_WRITE32(sinfo, CMICM_DMA_CTRLr + 4 * chan, cdc); 1345 1346 MEMORY_BARRIER; 1347 } 1348 1349 static inline void 1350 xgsm_dma_desc_clear(bkn_switch_info_t *sinfo, int chan) 1351 { 1352 uint32_t val; 1353 1354 val = DS_CMC_DESCRD_CMPLT_CLR(chan); 1355 if (CDMA_CH(sinfo, chan)) { 1356 val |= DS_CMC_CTRLD_INT_CLR(chan); 1357 } 1358 DEV_WRITE32(sinfo, CMICM_DMA_STAT_CLRr, val); 1359 1360 MEMORY_BARRIER; 1361 1362 /* Flush write buffer */ 1363 DEV_READ32(sinfo, CMICM_DMA_STAT_CLRr, &val); 1364 1365 MEMORY_BARRIER; 1366 } 1367 1368 static int 1369 xgsm_dma_chan_clear(bkn_switch_info_t *sinfo, int chan) 1370 { 1371 xgsm_dma_chain_clear(sinfo, chan); 1372 xgsm_dma_desc_clear(sinfo, chan); 1373 1374 return 0; 1375 } 1376 1377 static inline void 1378 xgsm_cdma_halt_set(bkn_switch_info_t *sinfo, int chan) 1379 { 1380 DEV_WRITE32(sinfo, CMICM_DMA_HALT_ADDRr + 4 * chan, 1381 sinfo->halt_addr[chan]); 1382 1383 MEMORY_BARRIER; 1384 } 1385 1386 static int 1387 xgsm_dma_chan_init(bkn_switch_info_t *sinfo, int chan, int dir) 1388 { 1389 uint32_t cdc; 1390 1391 DEV_READ32(sinfo, CMICM_DMA_CTRLr + 4 * chan, &cdc); 1392 cdc &= ~DC_CMC_DIRECTION; 1393 if (dir) { 1394 cdc |= DC_CMC_DIRECTION; 1395 } 1396 if (CDMA_CH(sinfo, chan)) { 1397 cdc |= DC_CMC_CONTINUOUS | DC_CMC_CTRLD_INT; 1398 xgsm_cdma_halt_set(sinfo, chan); 1399 } 1400 DEV_WRITE32(sinfo, CMICM_DMA_CTRLr + 4 * chan, cdc); 1401 1402 return 0; 1403 } 1404 1405 static int 1406 xgsm_dma_chan_start(bkn_switch_info_t *sinfo, int chan, uint64_t dcb) 1407 { 1408 uint32_t cdc; 1409 1410 /* Write the DCB address to the DESC address for this channel */ 1411 DEV_WRITE32(sinfo, CMICM_DMA_DESC0r + 4 * chan, dcb); 1412 1413 MEMORY_BARRIER; 1414 1415 /* Kick it off */ 1416 DEV_READ32(sinfo, CMICM_DMA_CTRLr + 4 * chan, &cdc); 1417 cdc |= DC_CMC_ENABLE; 1418 DEV_WRITE32(sinfo, CMICM_DMA_CTRLr + 4 * chan, cdc); 1419 1420 MEMORY_BARRIER; 1421 1422 return 0; 1423 } 1424 1425 static int 1426 xgsm_dma_chan_abort(bkn_switch_info_t *sinfo, int chan, int polls) 1427 { 1428 uint32_t ctrl, dma_stat; 1429 int p; 1430 1431 /* Skip abort sequence if channel is not active */ 1432 DEV_READ32(sinfo, CMICM_DMA_STATr, &dma_stat); 1433 if (!(dma_stat & DS_CMC_DMA_ACTIVE(chan))) { 1434 return 0; 1435 } 1436 1437 /* Abort the channel */ 1438 DEV_READ32(sinfo, CMICM_DMA_CTRLr + 4 * chan, &ctrl); 1439 ctrl |= (DC_CMC_ENABLE | DC_CMC_ABORT); 1440 DEV_WRITE32(sinfo, CMICM_DMA_CTRLr + 4 * chan, ctrl); 1441 1442 MEMORY_BARRIER; 1443 1444 /* Poll for abort completion */ 1445 for (p = 0; p < polls; p++) { 1446 DEV_READ32(sinfo, CMICM_DMA_STATr, &dma_stat); 1447 if (!(dma_stat & DS_CMC_DMA_ACTIVE(chan))) { 1448 /* Clear up channel */ 1449 xgsm_dma_chan_clear(sinfo, chan); 1450 return polls; 1451 } 1452 } 1453 DBG_WARN(("DMA channel %d abort failed\n", chan)); 1454 1455 return -1; 1456 } 1457 1458 static inline void 1459 xgsm_irq_mask_set(bkn_switch_info_t *sinfo, uint32_t mask) 1460 { 1461 uint32_t irq_mask_reg = CMICM_IRQ_PCI_MASKr; 1462 uint32_t ctrld_mask = 0; 1463 1464 if (sinfo->napi_poll_mode) { 1465 mask = 0; 1466 } 1467 if (sinfo->cpu_no == 1) { 1468 irq_mask_reg = CMICM_IRQ_UC0_MASKr; 1469 } 1470 1471 /* Get the Controlled Interrupts mask for Continuous DMA mode */ 1472 ctrld_mask |= (sinfo->cdma_channels << 27) & CMICD_CTRLD_IRQ_MASK; 1473 1474 lkbde_irq_mask_set(sinfo->dev_no | LKBDE_ISR2_DEV, irq_mask_reg, 1475 mask, CMICM_TXRX_IRQ_MASK | ctrld_mask); 1476 } 1477 1478 static inline void 1479 xgsm_irq_mask_enable(bkn_switch_info_t *sinfo, int chan, int update_hw) 1480 { 1481 uint32_t mask; 1482 1483 if (CDMA_CH(sinfo, chan)) { 1484 mask = 0x08000000 << chan; 1485 } else { 1486 if (chan == XGS_DMA_TX_CHAN) { 1487 mask = 0x8000; 1488 } else { 1489 mask = 0xc000 >> (2 * chan); 1490 } 1491 } 1492 1493 sinfo->irq_mask |= mask; 1494 1495 if (update_hw) { 1496 xgsm_irq_mask_set(sinfo, sinfo->irq_mask); 1497 } 1498 } 1499 1500 static inline void 1501 xgsm_irq_mask_disable(bkn_switch_info_t *sinfo, int chan, int update_hw) 1502 { 1503 uint32_t mask; 1504 1505 if (CDMA_CH(sinfo, chan)) { 1506 mask = 0x08000000 << chan; 1507 } else { 1508 if (chan == XGS_DMA_TX_CHAN) { 1509 mask = 0x8000; 1510 } else { 1511 mask = 0xc000 >> (2 * chan); 1512 } 1513 } 1514 1515 sinfo->irq_mask &= ~mask; 1516 1517 if (update_hw) { 1518 xgsm_irq_mask_set(sinfo, sinfo->irq_mask); 1519 } 1520 } 1521 1522 static inline void 1523 xgsx_dma_chain_clear(bkn_switch_info_t *sinfo, int chan) 1524 { 1525 uint32_t ctrl, stat; 1526 1527 /* Disabing DMA clears chain done */ 1528 DEV_READ32(sinfo, CMICX_DMA_CTRLr + 0x80 * chan, &ctrl); 1529 ctrl &= ~(CMICX_DC_CMC_ENABLE | CMICX_DC_CMC_ABORT); 1530 DEV_WRITE32(sinfo, CMICX_DMA_CTRLr + 0x80 * chan, ctrl); 1531 1532 stat = CMICX_DS_CMC_CHAIN_DONE(chan); 1533 DEV_WRITE32(sinfo, CMICX_IRQ_STAT_CLRr, stat); 1534 1535 MEMORY_BARRIER; 1536 1537 /* Flush write buffer */ 1538 DEV_READ32(sinfo, CMICX_IRQ_STAT_CLRr, &stat); 1539 1540 MEMORY_BARRIER; 1541 } 1542 1543 static inline void 1544 xgsx_dma_desc_clear(bkn_switch_info_t *sinfo, int chan) 1545 { 1546 uint32_t stat; 1547 1548 if (CDMA_CH(sinfo, chan)) { 1549 stat = CMICX_DS_CMC_CTRLD_INT(chan); 1550 } else { 1551 stat = CMICX_DS_CMC_DESC_DONE(chan); 1552 } 1553 DEV_WRITE32(sinfo, CMICX_IRQ_STAT_CLRr, stat); 1554 1555 MEMORY_BARRIER; 1556 1557 /* Flush write buffer */ 1558 DEV_READ32(sinfo, CMICX_IRQ_STAT_CLRr, &stat); 1559 1560 MEMORY_BARRIER; 1561 } 1562 1563 static int 1564 xgsx_dma_chan_clear(bkn_switch_info_t *sinfo, int chan) 1565 { 1566 xgsx_dma_chain_clear(sinfo, chan); 1567 xgsx_dma_desc_clear(sinfo, chan); 1568 1569 return 0; 1570 } 1571 1572 static inline void 1573 xgsx_cdma_halt_set(bkn_switch_info_t *sinfo, int chan) 1574 { 1575 DEV_WRITE32(sinfo, CMICX_DMA_HALT_LOr + 0x80 * chan, 1576 sinfo->halt_addr[chan]); 1577 DEV_WRITE32(sinfo, CMICX_DMA_HALT_HIr + 0x80 * chan, 1578 DMA_TO_BUS_HI(sinfo->halt_addr[chan] >> 32)); 1579 1580 MEMORY_BARRIER; 1581 } 1582 1583 static int 1584 xgsx_dma_chan_init(bkn_switch_info_t *sinfo, int chan, int dir) 1585 { 1586 uint32_t ctrl; 1587 1588 DEV_READ32(sinfo, CMICX_DMA_CTRLr + 0x80 * chan, &ctrl); 1589 ctrl &= ~CMICX_DC_CMC_DIRECTION; 1590 if (dir) { 1591 ctrl |= CMICX_DC_CMC_DIRECTION; 1592 } 1593 if (CDMA_CH(sinfo, chan)) { 1594 ctrl |= CMICX_DC_CMC_CONTINUOUS | CMICX_DC_CMC_CTRLD_INT; 1595 xgsx_cdma_halt_set(sinfo, chan); 1596 } 1597 DEV_WRITE32(sinfo, CMICX_DMA_CTRLr + 0x80 * chan, ctrl); 1598 1599 MEMORY_BARRIER; 1600 1601 return 0; 1602 } 1603 1604 static int 1605 xgsx_dma_chan_start(bkn_switch_info_t *sinfo, int chan, uint64_t dcb) 1606 { 1607 uint32_t ctrl; 1608 1609 /* Write the DCB address to the DESC address for this channel */ 1610 DEV_WRITE32(sinfo, CMICX_DMA_DESC_LOr + 0x80 * chan, dcb); 1611 DEV_WRITE32(sinfo, CMICX_DMA_DESC_HIr + 0x80 * chan, DMA_TO_BUS_HI(dcb >> 32)); 1612 1613 MEMORY_BARRIER; 1614 1615 /* Kick it off */ 1616 DEV_READ32(sinfo, CMICX_DMA_CTRLr + 0x80 * chan, &ctrl); 1617 ctrl |= CMICX_DC_CMC_ENABLE; 1618 DEV_WRITE32(sinfo, CMICX_DMA_CTRLr + 0x80 * chan, ctrl); 1619 1620 MEMORY_BARRIER; 1621 1622 return 0; 1623 } 1624 1625 static int 1626 xgsx_dma_chan_abort(bkn_switch_info_t *sinfo, int chan, int polls) 1627 { 1628 uint32_t ctrl, stat; 1629 int p; 1630 1631 /* Skip abort sequence if channel is not active */ 1632 DEV_READ32(sinfo, CMICX_DMA_STATr + 0x80 * chan, &stat); 1633 if (!(stat & CMICX_DS_CMC_DMA_ACTIVE)) { 1634 return 0; 1635 } 1636 1637 /* Abort the channel */ 1638 DEV_READ32(sinfo, CMICX_DMA_CTRLr + 0x80 * chan, &ctrl); 1639 ctrl |= CMICX_DC_CMC_ENABLE | CMICX_DC_CMC_ABORT; 1640 DEV_WRITE32(sinfo, CMICX_DMA_CTRLr + 0x80 * chan, ctrl); 1641 1642 MEMORY_BARRIER; 1643 1644 /* Poll for abort completion */ 1645 for (p = 0; p < polls; p++) { 1646 DEV_READ32(sinfo, CMICX_DMA_STATr + 0x80 * chan, &stat); 1647 if (!(stat & CMICX_DS_CMC_DMA_ACTIVE)) { 1648 /* Clear up channel */ 1649 xgsx_dma_chan_clear(sinfo, chan); 1650 return polls; 1651 } 1652 } 1653 1654 DBG_WARN(("DMA channel %d abort failed\n", chan)); 1655 1656 return -1; 1657 } 1658 1659 static inline void 1660 xgsx_irq_mask_set(bkn_switch_info_t *sinfo, uint32_t mask) 1661 { 1662 uint32_t irq_mask_reg = CMICX_IRQ_ENABr; 1663 uint32_t irq_mask, irq_fmask, disable_mask; 1664 1665 if (sinfo->napi_poll_mode) { 1666 mask = 0; 1667 } 1668 1669 if (sinfo->cpu_no == 1) { 1670 lkbde_irq_mask_get(sinfo->dev_no, &irq_mask, &irq_fmask); 1671 disable_mask = ~mask & (irq_mask & irq_fmask); 1672 if (disable_mask) { 1673 lkbde_irq_mask_set(sinfo->dev_no | LKBDE_ISR2_DEV | LKBDE_IPROC_REG, 1674 IHOST_GIC_GIC400_GICD_ICENABLERN_5r, disable_mask, CMICX_TXRX_IRQ_MASK); 1675 } 1676 irq_mask_reg = IHOST_GIC_GIC400_GICD_ISENABLERN_5r; 1677 } 1678 1679 lkbde_irq_mask_set(sinfo->dev_no | LKBDE_ISR2_DEV | LKBDE_IPROC_REG, 1680 irq_mask_reg, mask, CMICX_TXRX_IRQ_MASK); 1681 } 1682 1683 static inline void 1684 xgsx_irq_mask_enable(bkn_switch_info_t *sinfo, int chan, int update_hw) 1685 { 1686 uint32_t mask; 1687 1688 if (CDMA_CH(sinfo, chan)) { 1689 mask = CMICX_DS_CMC_CTRLD_INT(chan); 1690 } else { 1691 if (chan == XGS_DMA_TX_CHAN) { 1692 mask = CMICX_DS_CMC_CHAIN_DONE(chan); 1693 } else { 1694 mask = CMICX_DS_CMC_DESC_DONE(chan) | 1695 CMICX_DS_CMC_CHAIN_DONE(chan); 1696 } 1697 } 1698 1699 sinfo->irq_mask |= mask; 1700 1701 if (update_hw) { 1702 xgsx_irq_mask_set(sinfo, sinfo->irq_mask); 1703 } 1704 } 1705 1706 static inline void 1707 xgsx_irq_mask_disable(bkn_switch_info_t *sinfo, int chan, int update_hw) 1708 { 1709 uint32_t mask; 1710 1711 if (CDMA_CH(sinfo, chan)) { 1712 mask = CMICX_DS_CMC_CTRLD_INT(chan); 1713 } else { 1714 if (chan == XGS_DMA_TX_CHAN) { 1715 mask = CMICX_DS_CMC_CHAIN_DONE(chan); 1716 } else { 1717 mask = CMICX_DS_CMC_DESC_DONE(chan) | 1718 CMICX_DS_CMC_CHAIN_DONE(chan); 1719 } 1720 } 1721 1722 sinfo->irq_mask &= ~mask; 1723 1724 if (update_hw) { 1725 xgsx_irq_mask_set(sinfo, sinfo->irq_mask); 1726 } 1727 } 1728 1729 static inline void 1730 dev_dma_chain_clear(bkn_switch_info_t *sinfo, int chan) 1731 { 1732 if (DEV_IS_CMICX(sinfo)) { 1733 xgsx_dma_chain_clear(sinfo, chan); 1734 } else if (DEV_IS_CMICM(sinfo)) { 1735 xgsm_dma_chain_clear(sinfo, chan); 1736 } else { 1737 xgs_dma_chain_clear(sinfo, chan); 1738 } 1739 } 1740 1741 static inline void 1742 dev_dma_desc_clear(bkn_switch_info_t *sinfo, int chan) 1743 { 1744 if (DEV_IS_CMICX(sinfo)) { 1745 xgsx_dma_desc_clear(sinfo, chan); 1746 } else if (DEV_IS_CMICM(sinfo)) { 1747 xgsm_dma_desc_clear(sinfo, chan); 1748 } else { 1749 xgs_dma_desc_clear(sinfo, chan); 1750 } 1751 } 1752 1753 static int 1754 dev_dma_chan_clear(bkn_switch_info_t *sinfo, int chan) 1755 { 1756 if (DEV_IS_CMICX(sinfo)) { 1757 return xgsx_dma_chan_clear(sinfo, chan); 1758 } else if (DEV_IS_CMICM(sinfo)) { 1759 return xgsm_dma_chan_clear(sinfo, chan); 1760 } else { 1761 return xgs_dma_chan_clear(sinfo, chan); 1762 } 1763 } 1764 1765 static void 1766 dev_cdma_halt_set(bkn_switch_info_t *sinfo, int chan) 1767 { 1768 if (DEV_IS_CMICX(sinfo)) { 1769 xgsx_cdma_halt_set(sinfo, chan); 1770 } else if (DEV_IS_CMICM(sinfo)) { 1771 xgsm_cdma_halt_set(sinfo, chan); 1772 } 1773 } 1774 1775 static int 1776 dev_dma_chan_init(bkn_switch_info_t *sinfo, int chan, int dir) 1777 { 1778 if (DEV_IS_CMICX(sinfo)) { 1779 return xgsx_dma_chan_init(sinfo, chan, dir); 1780 } else if (DEV_IS_CMICM(sinfo)) { 1781 return xgsm_dma_chan_init(sinfo, chan, dir); 1782 } else { 1783 return xgs_dma_chan_init(sinfo, chan, dir); 1784 } 1785 } 1786 1787 static int 1788 dev_dma_chan_start(bkn_switch_info_t *sinfo, int chan, uint64_t dcb) 1789 { 1790 if (DEV_IS_CMICX(sinfo)) { 1791 return xgsx_dma_chan_start(sinfo, chan, dcb); 1792 } else if (DEV_IS_CMICM(sinfo)) { 1793 return xgsm_dma_chan_start(sinfo, chan, dcb); 1794 } else { 1795 return xgs_dma_chan_start(sinfo, chan, dcb); 1796 } 1797 } 1798 1799 static int 1800 dev_dma_chan_abort(bkn_switch_info_t *sinfo, int chan, int polls) 1801 { 1802 if (DEV_IS_CMICX(sinfo)) { 1803 return xgsx_dma_chan_abort(sinfo, chan, polls); 1804 } else if (DEV_IS_CMICM(sinfo)) { 1805 return xgsm_dma_chan_abort(sinfo, chan, polls); 1806 } else { 1807 return xgs_dma_chan_abort(sinfo, chan, polls); 1808 } 1809 } 1810 1811 static inline void 1812 dev_irq_mask_set(bkn_switch_info_t *sinfo, uint32_t mask) 1813 { 1814 if (DEV_IS_CMICX(sinfo)) { 1815 xgsx_irq_mask_set(sinfo, mask); 1816 } else if (DEV_IS_CMICM(sinfo)) { 1817 xgsm_irq_mask_set(sinfo, mask); 1818 } else { 1819 xgs_irq_mask_set(sinfo, mask); 1820 } 1821 } 1822 1823 static inline void 1824 dev_irq_mask_enable(bkn_switch_info_t *sinfo, int chan, int update_hw) 1825 { 1826 if (DEV_IS_CMICX(sinfo)) { 1827 xgsx_irq_mask_enable(sinfo, chan, update_hw); 1828 } else if (DEV_IS_CMICM(sinfo)) { 1829 xgsm_irq_mask_enable(sinfo, chan, update_hw); 1830 } else { 1831 xgs_irq_mask_enable(sinfo, chan, update_hw); 1832 } 1833 } 1834 1835 static void 1836 dev_irq_mask_disable(bkn_switch_info_t *sinfo, int chan, int update_hw) 1837 { 1838 if (DEV_IS_CMICX(sinfo)) { 1839 xgsx_irq_mask_disable(sinfo, chan, update_hw); 1840 } else if (DEV_IS_CMICM(sinfo)) { 1841 xgsm_irq_mask_disable(sinfo, chan, update_hw); 1842 } else { 1843 xgs_irq_mask_disable(sinfo, chan, update_hw); 1844 } 1845 } 1846 1847 static int 1848 bkn_alloc_dcbs(bkn_switch_info_t *sinfo) 1849 { 1850 int dcb_size; 1851 int tx_ring_size, rx_ring_size; 1852 dma_addr_t dcb_dma = 0; 1853 1854 dcb_size = sinfo->dcb_wsize * sizeof(uint32_t); 1855 tx_ring_size = dcb_size * (MAX_TX_DCBS + 1); 1856 rx_ring_size = dcb_size * (MAX_RX_DCBS + 1); 1857 sinfo->dcb_mem_size = tx_ring_size + rx_ring_size * sinfo->rx_chans; 1858 1859 sinfo->dcb_mem = DMA_ALLOC_COHERENT(sinfo->dma_dev, 1860 sinfo->dcb_mem_size, 1861 &dcb_dma); 1862 if (sinfo->dcb_mem == NULL) { 1863 gprintk("DCB memory allocation (%d bytes) failed.\n", 1864 sinfo->dcb_mem_size); 1865 return -ENOMEM; 1866 } 1867 sinfo->dcb_dma = (uint64_t)dcb_dma; 1868 1869 return 0; 1870 } 1871 1872 static void 1873 bkn_free_dcbs(bkn_switch_info_t *sinfo) 1874 { 1875 if (sinfo->dcb_mem != NULL) { 1876 DMA_FREE_COHERENT(sinfo->dma_dev, sinfo->dcb_mem_size, 1877 sinfo->dcb_mem, (dma_addr_t)sinfo->dcb_dma); 1878 sinfo->dcb_mem = NULL; 1879 } 1880 } 1881 1882 static void 1883 bkn_clean_tx_dcbs(bkn_switch_info_t *sinfo) 1884 { 1885 bkn_desc_info_t *desc; 1886 1887 DBG_DCB_TX(("Cleaning Tx DCBs (%d %d).\n", 1888 sinfo->tx.cur, sinfo->tx.dirty)); 1889 while (sinfo->tx.free < MAX_TX_DCBS) { 1890 desc = &sinfo->tx.desc[sinfo->tx.dirty]; 1891 if (desc->skb != NULL) { 1892 DBG_SKB(("Cleaning Tx SKB from DCB %d.\n", 1893 sinfo->tx.dirty)); 1894 DMA_UNMAP_SINGLE(sinfo->dma_dev, 1895 desc->skb_dma, desc->dma_size, 1896 DMA_TODEV); 1897 desc->skb_dma = 0; 1898 dev_kfree_skb_any(desc->skb); 1899 desc->skb = NULL; 1900 } 1901 if (++sinfo->tx.dirty >= MAX_TX_DCBS) { 1902 sinfo->tx.dirty = 0; 1903 } 1904 sinfo->tx.free++; 1905 } 1906 sinfo->tx.api_active = 0; 1907 DBG_DCB_TX(("Cleaned Tx DCBs (%d %d).\n", 1908 sinfo->tx.cur, sinfo->tx.dirty)); 1909 } 1910 1911 static void 1912 bkn_clean_rx_dcbs(bkn_switch_info_t *sinfo, int chan) 1913 { 1914 bkn_desc_info_t *desc; 1915 1916 DBG_DCB_RX(("Cleaning Rx%d DCBs (%d %d).\n", 1917 chan, sinfo->rx[chan].cur, sinfo->rx[chan].dirty)); 1918 while (sinfo->rx[chan].free) { 1919 desc = &sinfo->rx[chan].desc[sinfo->rx[chan].dirty]; 1920 if (desc->skb != NULL) { 1921 DBG_SKB(("Cleaning Rx%d SKB from DCB %d.\n", 1922 chan, sinfo->rx[chan].dirty)); 1923 DMA_UNMAP_SINGLE(sinfo->dma_dev, 1924 desc->skb_dma, desc->dma_size, 1925 DMA_FROMDEV); 1926 desc->skb_dma = 0; 1927 dev_kfree_skb_any(desc->skb); 1928 desc->skb = NULL; 1929 } 1930 if (++sinfo->rx[chan].dirty >= MAX_RX_DCBS) { 1931 sinfo->rx[chan].dirty = 0; 1932 } 1933 sinfo->rx[chan].free--; 1934 } 1935 sinfo->rx[chan].running = 0; 1936 sinfo->rx[chan].api_active = 0; 1937 DBG_DCB_RX(("Cleaned Rx%d DCBs (%d %d).\n", 1938 chan, sinfo->rx[chan].cur, sinfo->rx[chan].dirty)); 1939 } 1940 1941 static void 1942 bkn_clean_dcbs(bkn_switch_info_t *sinfo) 1943 { 1944 int chan; 1945 1946 bkn_clean_tx_dcbs(sinfo); 1947 1948 for (chan = 0; chan < sinfo->rx_chans; chan++) { 1949 bkn_clean_rx_dcbs(sinfo, chan); 1950 } 1951 } 1952 1953 static void 1954 bkn_init_dcbs(bkn_switch_info_t *sinfo) 1955 { 1956 int dcb_size; 1957 uint32_t *dcb_mem; 1958 uint64_t dcb_dma; 1959 bkn_desc_info_t *desc; 1960 int idx; 1961 int chan; 1962 1963 memset(sinfo->dcb_mem, 0, sinfo->dcb_mem_size); 1964 1965 dcb_size = sinfo->dcb_wsize * sizeof(uint32_t); 1966 dcb_mem = sinfo->dcb_mem; 1967 dcb_dma = sinfo->dcb_dma; 1968 1969 for (idx = 0; idx < (MAX_TX_DCBS + 1); idx++) { 1970 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN)) { 1971 if (sinfo->cmic_type == 'x') { 1972 dcb_mem[2] |= 1 << 24 | 1 << 16; 1973 } else { 1974 dcb_mem[1] |= 1 << 24 | 1 << 16; 1975 } 1976 if (idx == MAX_TX_DCBS) { 1977 if (sinfo->cmic_type == 'x') { 1978 dcb_mem[0] = sinfo->tx.desc[0].dcb_dma; 1979 dcb_mem[1] = DMA_TO_BUS_HI(sinfo->tx.desc[0].dcb_dma >> 32); 1980 dcb_mem[2] |= 1 << 18; 1981 } else { 1982 dcb_mem[0] = sinfo->tx.desc[0].dcb_dma; 1983 dcb_mem[1] |= 1 << 18; 1984 } 1985 } 1986 } 1987 desc = &sinfo->tx.desc[idx]; 1988 desc->dcb_mem = dcb_mem; 1989 desc->dcb_dma = dcb_dma; 1990 dcb_mem += sinfo->dcb_wsize; 1991 dcb_dma += dcb_size; 1992 } 1993 sinfo->halt_addr[XGS_DMA_TX_CHAN] = sinfo->tx.desc[0].dcb_dma; 1994 sinfo->tx.free = MAX_TX_DCBS; 1995 sinfo->tx.cur = 0; 1996 sinfo->tx.dirty = 0; 1997 1998 DBG_DCB_TX(("Tx DCBs @ 0x%08x.\n", 1999 (uint32_t)sinfo->tx.desc[0].dcb_dma)); 2000 2001 for (chan = 0; chan < sinfo->rx_chans; chan++) { 2002 for (idx = 0; idx < (MAX_RX_DCBS + 1); idx++) { 2003 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 2004 if (sinfo->cmic_type == 'x') { 2005 dcb_mem[2] |= 1 << 24 | 1 << 16; 2006 } else { 2007 dcb_mem[1] |= 1 << 24 | 1 << 16; 2008 } 2009 if (idx == MAX_RX_DCBS) { 2010 if (sinfo->cmic_type == 'x') { 2011 dcb_mem[0] = sinfo->rx[chan].desc[0].dcb_dma; 2012 dcb_mem[1] = DMA_TO_BUS_HI(sinfo->rx[chan].desc[0].dcb_dma >> 32); 2013 dcb_mem[2] |= 1 << 18; 2014 } else { 2015 dcb_mem[0] = sinfo->rx[chan].desc[0].dcb_dma; 2016 dcb_mem[1] |= 1 << 18; 2017 } 2018 } 2019 } 2020 desc = &sinfo->rx[chan].desc[idx]; 2021 desc->dcb_mem = dcb_mem; 2022 desc->dcb_dma = dcb_dma; 2023 dcb_mem += sinfo->dcb_wsize; 2024 dcb_dma += dcb_size; 2025 } 2026 sinfo->halt_addr[XGS_DMA_RX_CHAN + chan] = sinfo->rx[chan].desc[MAX_RX_DCBS].dcb_dma; 2027 sinfo->rx[chan].free = 0; 2028 sinfo->rx[chan].cur = 0; 2029 sinfo->rx[chan].dirty = 0; 2030 2031 DBG_DCB_RX(("Rx%d DCBs @ 0x%08x.\n", 2032 chan, (uint32_t)sinfo->rx[chan].desc[0].dcb_dma)); 2033 } 2034 } 2035 2036 static void 2037 bkn_dump_dcb(char *prefix, uint32_t *dcb, int wsize, int txrx) 2038 { 2039 if (XGS_DMA_TX_CHAN == txrx) { 2040 if (wsize > 4) { 2041 DBG_DCB_TX(("%s: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x ... 0x%08x\n", 2042 prefix, dcb[0], dcb[1], dcb[2], dcb[3], dcb[4], dcb[5], 2043 dcb[wsize - 1])); 2044 } else { 2045 DBG_DCB_TX(("%s: 0x%08x 0x%08x 0x%08x 0x%08x\n", 2046 prefix, dcb[0], dcb[1], dcb[2], dcb[3])); 2047 } 2048 } else { 2049 if (wsize > 4) { 2050 DBG_DCB_RX(("%s: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x ... 0x%08x\n", 2051 prefix, dcb[0], dcb[1], dcb[2], dcb[3], dcb[4], dcb[5], 2052 dcb[wsize - 1])); 2053 } else { 2054 DBG_DCB_RX(("%s: 0x%08x 0x%08x 0x%08x 0x%08x\n", 2055 prefix, dcb[0], dcb[1], dcb[2], dcb[3])); 2056 } 2057 } 2058 } 2059 2060 static void 2061 bkn_dump_pkt(uint8_t *data, int size, int txrx) 2062 { 2063 int idx; 2064 char str[128]; 2065 2066 if (!(debug & DBG_LVL_PDMP) && 2067 !(txrx == XGS_DMA_TX_CHAN && debug & DBG_LVL_PDMP_TX) && 2068 !(txrx == XGS_DMA_RX_CHAN && debug & DBG_LVL_PDMP_RX)) { 2069 return; 2070 } 2071 2072 for (idx = 0; idx < size; idx++) { 2073 if ((idx & 0xf) == 0) { 2074 sprintf(str, "%04x: ", idx); 2075 } 2076 if ((idx & 0xf) == 8) { 2077 sprintf(&str[strlen(str)], "- "); 2078 } 2079 sprintf(&str[strlen(str)], "%02x ", data[idx]); 2080 if ((idx & 0xf) == 0xf) { 2081 sprintf(&str[strlen(str)], "\n"); 2082 gprintk(str); 2083 } 2084 } 2085 if ((idx & 0xf) != 0) { 2086 sprintf(&str[strlen(str)], "\n"); 2087 gprintk(str); 2088 } 2089 } 2090 2091 static bkn_switch_info_t * 2092 bkn_sinfo_from_unit(int unit) 2093 { 2094 struct list_head *list; 2095 bkn_switch_info_t *sinfo; 2096 2097 list_for_each(list, &_sinfo_list) { 2098 sinfo = (bkn_switch_info_t *)list; 2099 if (sinfo->dev_no == unit || unit == -1) { 2100 return sinfo; 2101 } 2102 } 2103 return NULL; 2104 } 2105 2106 static void 2107 bkn_cdma_goto(bkn_switch_info_t *sinfo, int chan, uint64_t dcb) 2108 { 2109 if (sinfo->basedev_suspended) { 2110 return; 2111 } 2112 2113 /* Set the new halt location */ 2114 sinfo->halt_addr[chan] = dcb; 2115 dev_cdma_halt_set(sinfo, chan); 2116 } 2117 2118 static void 2119 bkn_api_rx_restart(bkn_switch_info_t *sinfo, int chan) 2120 { 2121 bkn_dcb_chain_t *dcb_chain; 2122 int start_dma; 2123 2124 if (sinfo->basedev_suspended) { 2125 return; 2126 } 2127 2128 /* If resume from basedev suspended, there could be a suspended chain */ 2129 if (sinfo->rx[chan].api_dcb_chain) { 2130 return; 2131 } 2132 2133 /* Assume that driver lock is held */ 2134 if (!list_empty(&sinfo->rx[chan].api_dcb_list)) { 2135 dcb_chain = list_entry(sinfo->rx[chan].api_dcb_list.next, 2136 bkn_dcb_chain_t, list); 2137 start_dma = 0; 2138 if (sinfo->rx[chan].use_rx_skb == 0) { 2139 sinfo->rx[chan].chain_complete = 0; 2140 start_dma = 1; 2141 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan) && 2142 sinfo->rx[chan].api_active) { 2143 /* HW is running already, so we just move to the next chain */ 2144 start_dma = 0; 2145 } 2146 } 2147 sinfo->rx[chan].api_dcb_chain = dcb_chain; 2148 sinfo->rx[chan].api_active = 1; 2149 if (start_dma) { 2150 DBG_DCB_RX(("Start API Rx DMA, first DCB @ 0x%08x (%d DCBs).\n", 2151 (uint32_t)dcb_chain->dcb_dma, dcb_chain->dcb_cnt)); 2152 dev_dma_chan_clear(sinfo, XGS_DMA_RX_CHAN + chan); 2153 dev_irq_mask_enable(sinfo, XGS_DMA_RX_CHAN + chan, 1); 2154 dev_dma_chan_start(sinfo, XGS_DMA_RX_CHAN + chan, dcb_chain->dcb_dma); 2155 } 2156 2157 list_del(&dcb_chain->list); 2158 } 2159 } 2160 2161 static void 2162 bkn_api_rx_chain_done(bkn_switch_info_t *sinfo, int chan) 2163 { 2164 DBG_DCB_RX(("API Rx DMA chain done\n")); 2165 2166 if (!CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 2167 sinfo->rx[chan].api_active = 0; 2168 } 2169 if (sinfo->rx[chan].api_dcb_chain) { 2170 kfree(sinfo->rx[chan].api_dcb_chain); 2171 sinfo->rx[chan].api_dcb_chain = NULL; 2172 } 2173 bkn_api_rx_restart(sinfo, chan); 2174 } 2175 2176 static int 2177 bkn_api_rx_copy_from_skb(bkn_switch_info_t *sinfo, 2178 int chan, bkn_desc_info_t *desc) 2179 { 2180 bkn_dcb_chain_t *dcb_chain; 2181 uint32_t *dcb; 2182 uint32_t dcb_stat; 2183 uint8_t *pkt; 2184 uint64_t pkt_dma; 2185 int pktlen; 2186 int i; 2187 bkn_evt_resource_t *evt; 2188 2189 dcb_stat = desc->dcb_mem[sinfo->dcb_wsize-1]; 2190 pktlen = dcb_stat & SOC_DCB_KNET_COUNT_MASK; 2191 2192 dcb_chain = sinfo->rx[chan].api_dcb_chain; 2193 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan) && dcb_chain == NULL) { 2194 /* Last chain done, try to get a new one */ 2195 bkn_api_rx_chain_done(sinfo, chan); 2196 dcb_chain = sinfo->rx[chan].api_dcb_chain; 2197 } 2198 if (dcb_chain == NULL) { 2199 DBG_WARN(("No Rx API buffers\n")); 2200 sinfo->rx[chan].pkts_d_no_api_buf++; 2201 return -1; 2202 } 2203 dcb = &dcb_chain->dcb_mem[dcb_chain->dcb_cur * sinfo->dcb_wsize]; 2204 if ((sinfo->cmic_type == 'x' && (dcb[2] & 0xffff) < pktlen) || 2205 (sinfo->cmic_type != 'x' && (dcb[1] & 0xffff) < pktlen)) { 2206 DBG_WARN(("Rx API buffer too small\n")); 2207 return -1; 2208 } 2209 if (sinfo->cmic_type == 'x') { 2210 pkt_dma = BUS_TO_DMA_HI(dcb[1]); 2211 pkt_dma = pkt_dma << 32 | dcb[0]; 2212 } else { 2213 pkt_dma = dcb[0]; 2214 } 2215 pkt = (uint8_t *)kernel_bde->p2l(sinfo->dev_no, (sal_paddr_t)pkt_dma); 2216 if (pkt == NULL) { 2217 DBG_WARN(("Invalid Rx API buffer\n")); 2218 return -1; 2219 } 2220 2221 /* Copy packet data */ 2222 memcpy(pkt, desc->skb->data, pktlen); 2223 2224 /* Copy packet metadata and mark as done */ 2225 if (sinfo->cmic_type != 'x') { 2226 for (i = SOC_DCB_META_OFFSET; i < sinfo->dcb_wsize; i++) { 2227 dcb[i] = desc->dcb_mem[i]; 2228 } 2229 } 2230 dcb[sinfo->dcb_wsize-1] = dcb_stat | SOC_DCB_KNET_DONE; 2231 MEMORY_BARRIER; 2232 2233 dcb_chain->dcb_cur++; 2234 2235 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 2236 dcb = &dcb_chain->dcb_mem[dcb_chain->dcb_cur * sinfo->dcb_wsize]; 2237 if ((sinfo->cmic_type == 'x' && dcb[2] & (1 << 18)) || 2238 (sinfo->cmic_type != 'x' && dcb[1] & (1 << 18))) { 2239 /* Get the next chain if reload done */ 2240 dcb[sinfo->dcb_wsize-1] |= 1 << 31 | SOC_DCB_KNET_DONE; 2241 MEMORY_BARRIER; 2242 bkn_api_rx_chain_done(sinfo, chan); 2243 dcb_chain = sinfo->rx[chan].api_dcb_chain; 2244 if (dcb_chain == NULL) { 2245 sinfo->rx[chan].api_dcb_chain_end = NULL; 2246 } 2247 } 2248 } else { 2249 if ((sinfo->cmic_type == 'x' && (dcb[2] & (1 << 16)) == 0) || 2250 (sinfo->cmic_type != 'x' && (dcb[1] & (1 << 16)) == 0)) { 2251 bkn_api_rx_chain_done(sinfo, chan); 2252 } 2253 } 2254 2255 sinfo->dma_events |= KCOM_DMA_INFO_F_RX_DONE; 2256 2257 evt = &_bkn_evt[sinfo->evt_idx]; 2258 evt->evt_wq_put++; 2259 wake_up_interruptible(&evt->evt_wq); 2260 2261 return 0; 2262 } 2263 2264 static void 2265 bkn_rx_refill(bkn_switch_info_t *sinfo, int chan) 2266 { 2267 struct sk_buff *skb; 2268 bkn_desc_info_t *desc; 2269 uint32_t *dcb; 2270 uint32_t resv_size = sinfo->cmic_type == 'x' ? RCPU_HDR_SIZE : RCPU_RX_ENCAP_SIZE; 2271 uint32_t meta_size = sinfo->cmic_type == 'x' ? RCPU_RX_META_SIZE : 0; 2272 int prev; 2273 2274 if (sinfo->rx[chan].use_rx_skb == 0) { 2275 /* Rx buffers are provided by BCM Rx API */ 2276 return; 2277 } 2278 2279 if (!CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan) && 2280 sinfo->rx[chan].tokens < MAX_RX_DCBS) { 2281 /* Pause DMA for now */ 2282 return; 2283 } 2284 2285 while (sinfo->rx[chan].free < MAX_RX_DCBS) { 2286 desc = &sinfo->rx[chan].desc[sinfo->rx[chan].cur]; 2287 if (desc->skb == NULL) { 2288 skb = dev_alloc_skb(rx_buffer_size + RCPU_RX_ENCAP_SIZE); 2289 if (skb == NULL) { 2290 break; 2291 } 2292 skb_reserve(skb, resv_size); 2293 desc->skb = skb; 2294 } else { 2295 DBG_DCB_RX(("Refill Rx%d SKB in DCB %d recycled.\n", 2296 chan, sinfo->rx[chan].cur)); 2297 } 2298 skb = desc->skb; 2299 desc->dma_size = rx_buffer_size + meta_size; 2300 #ifdef KNET_NO_AXI_DMA_INVAL 2301 2302 /* 2303 * Cache invalidate may corrupt DMA memory on some iProc-based devices 2304 * if the kernel is mapped to ACP memory. 2305 */ 2306 if (sinfo->pdev == NULL) { 2307 desc->dma_size = 0; 2308 } 2309 #endif 2310 desc->skb_dma = DMA_MAP_SINGLE(sinfo->dma_dev, 2311 skb->data, desc->dma_size, 2312 DMA_FROMDEV); 2313 if (DMA_MAPPING_ERROR(sinfo->dma_dev, desc->skb_dma)) { 2314 dev_kfree_skb_any(skb); 2315 desc->skb = NULL; 2316 break; 2317 } 2318 DBG_DCB_RX(("Refill Rx%d DCB %d (0x%08x).\n", 2319 chan, sinfo->rx[chan].cur, (uint32_t)desc->skb_dma)); 2320 dcb = desc->dcb_mem; 2321 dcb[0] = desc->skb_dma; 2322 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 2323 if (sinfo->cmic_type == 'x') { 2324 dcb[2] |= 1 << 24 | 1 << 16; 2325 } else { 2326 dcb[1] |= 1 << 24 | 1 << 16; 2327 } 2328 } else { 2329 prev = PREV_IDX(sinfo->rx[chan].cur, MAX_RX_DCBS); 2330 if (prev < (MAX_RX_DCBS - 1)) { 2331 if (sinfo->cmic_type == 'x') { 2332 sinfo->rx[chan].desc[prev].dcb_mem[2] |= 1 << 16; 2333 } else { 2334 sinfo->rx[chan].desc[prev].dcb_mem[1] |= 1 << 16; 2335 } 2336 } 2337 } 2338 if (sinfo->cmic_type == 'x') { 2339 dcb[1] = DMA_TO_BUS_HI(desc->skb_dma >> 32); 2340 dcb[2] |= rx_buffer_size + meta_size; 2341 } else { 2342 dcb[1] |= rx_buffer_size; 2343 } 2344 2345 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan) && 2346 sinfo->rx[chan].tokens > MAX_RX_DCBS) { 2347 /* DMA run to the new halt location */ 2348 bkn_cdma_goto(sinfo, XGS_DMA_RX_CHAN + chan, desc->dcb_dma); 2349 } 2350 2351 if (++sinfo->rx[chan].cur >= MAX_RX_DCBS) { 2352 sinfo->rx[chan].cur = 0; 2353 } 2354 sinfo->rx[chan].free++; 2355 sinfo->rx[chan].tokens--; 2356 } 2357 } 2358 2359 static int 2360 bkn_rx_restart(bkn_switch_info_t *sinfo, int chan) 2361 { 2362 bkn_desc_info_t *desc; 2363 2364 if (sinfo->basedev_suspended) { 2365 return 0; 2366 } 2367 2368 if (sinfo->rx[chan].running) { 2369 return 0; 2370 } 2371 2372 if (sinfo->rx[chan].free < MAX_RX_DCBS) { 2373 return 1; 2374 } 2375 2376 desc = &sinfo->rx[chan].desc[sinfo->rx[chan].dirty]; 2377 sinfo->rx[chan].chain_complete = 0; 2378 DBG_DCB_RX(("Restart Rx%d DMA, DCB @ 0x%08x (%d).\n", 2379 chan, (uint32_t)desc->dcb_dma, sinfo->rx[chan].dirty)); 2380 dev_dma_chan_clear(sinfo, XGS_DMA_RX_CHAN + chan); 2381 dev_irq_mask_enable(sinfo, XGS_DMA_RX_CHAN + chan, 1); 2382 dev_dma_chan_start(sinfo, XGS_DMA_RX_CHAN + chan, desc->dcb_dma); 2383 sinfo->rx[chan].running = 1; 2384 2385 /* Request one extra poll if chain was restarted during poll */ 2386 if (sinfo->napi_poll_mode) { 2387 sinfo->napi_poll_again = 1; 2388 } 2389 2390 return 0; 2391 } 2392 2393 static int 2394 bkn_tx_dma_start(bkn_switch_info_t *sinfo) 2395 { 2396 bkn_desc_info_t *desc; 2397 2398 desc = &sinfo->tx.desc[sinfo->tx.cur]; 2399 if (sinfo->tx.free == MAX_TX_DCBS) { 2400 if (!sinfo->tx.api_active) { 2401 DBG_DCB_TX(("Start Tx DMA, DCB @ 0x%08x (%d).\n", 2402 (uint32_t)desc->dcb_dma, sinfo->tx.cur)); 2403 dev_dma_chan_clear(sinfo, XGS_DMA_TX_CHAN); 2404 dev_irq_mask_enable(sinfo, XGS_DMA_TX_CHAN, 1); 2405 dev_dma_chan_start(sinfo, XGS_DMA_TX_CHAN, desc->dcb_dma); 2406 } 2407 } 2408 2409 return 0; 2410 } 2411 2412 static int 2413 bkn_dma_init(bkn_switch_info_t *sinfo) 2414 { 2415 int chan; 2416 2417 dev_dma_chan_init(sinfo, XGS_DMA_TX_CHAN, 1); 2418 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN)) { 2419 bkn_tx_dma_start(sinfo); 2420 } 2421 2422 for (chan = 0; chan < sinfo->rx_chans; chan++) { 2423 dev_dma_chan_init(sinfo, XGS_DMA_RX_CHAN + chan, 0); 2424 bkn_rx_refill(sinfo, chan); 2425 bkn_rx_restart(sinfo, chan); 2426 } 2427 2428 return 0; 2429 } 2430 2431 static int 2432 bkn_dma_abort_tx(bkn_switch_info_t *sinfo) 2433 { 2434 bkn_dcb_chain_t *dcb_chain; 2435 2436 DBG_VERB(("Aborting Tx DMA.\n")); 2437 2438 dev_irq_mask_disable(sinfo, XGS_DMA_TX_CHAN, 1); 2439 2440 dev_dma_chan_abort(sinfo, XGS_DMA_TX_CHAN, 10000); 2441 2442 if (sinfo->tx.api_dcb_chain) { 2443 DBG_DCB_TX(("Freeing active Tx DCB chain.\n")); 2444 kfree(sinfo->tx.api_dcb_chain); 2445 sinfo->tx.api_dcb_chain = NULL; 2446 } 2447 while (!list_empty(&sinfo->tx.api_dcb_list)) { 2448 dcb_chain = list_entry(sinfo->tx.api_dcb_list.next, 2449 bkn_dcb_chain_t, list); 2450 list_del(&dcb_chain->list); 2451 DBG_DCB_TX(("Freeing Tx DCB chain.\n")); 2452 kfree(dcb_chain); 2453 } 2454 sinfo->tx.api_dcb_chain_end = NULL; 2455 2456 return 0; 2457 } 2458 2459 static int 2460 bkn_dma_abort_rx(bkn_switch_info_t *sinfo, int chan) 2461 { 2462 bkn_dcb_chain_t *dcb_chain; 2463 2464 DBG_VERB(("Aborting Rx%d DMA.\n", chan)); 2465 2466 dev_irq_mask_disable(sinfo, XGS_DMA_RX_CHAN + chan, 1); 2467 2468 dev_dma_chan_abort(sinfo, XGS_DMA_RX_CHAN + chan, 10000); 2469 2470 if (sinfo->rx[chan].api_dcb_chain) { 2471 DBG_DCB_RX(("Freeing active Rx%d DCB chain.\n", chan)); 2472 kfree(sinfo->rx[chan].api_dcb_chain); 2473 sinfo->rx[chan].api_dcb_chain = NULL; 2474 } 2475 while (!list_empty(&sinfo->rx[chan].api_dcb_list)) { 2476 dcb_chain = list_entry(sinfo->rx[chan].api_dcb_list.next, 2477 bkn_dcb_chain_t, list); 2478 list_del(&dcb_chain->list); 2479 DBG_DCB_RX(("Freeing Rx%d DCB chain.\n", chan)); 2480 kfree(dcb_chain); 2481 } 2482 sinfo->rx[chan].api_dcb_chain_end = NULL; 2483 2484 return 0; 2485 } 2486 2487 static int 2488 bkn_dma_abort(bkn_switch_info_t *sinfo) 2489 { 2490 int chan; 2491 2492 bkn_dma_abort_tx(sinfo); 2493 2494 for (chan = 0; chan < sinfo->rx_chans; chan++) { 2495 bkn_dma_abort_rx(sinfo, chan); 2496 } 2497 2498 return 0; 2499 } 2500 2501 static int 2502 device_is_dpp(bkn_switch_info_t *sinfo) 2503 { 2504 int is_dpp = 0; 2505 2506 is_dpp = (sinfo->dcb_type == 28) ? 1 : 0; 2507 return is_dpp; 2508 } 2509 2510 static int 2511 device_is_dnx(bkn_switch_info_t *sinfo) 2512 { 2513 int is_dnx = 0; 2514 2515 /* No EP_TO_CPU header for DNX(JR2) */ 2516 is_dnx = ((sinfo->cmic_type == 'x') && (sinfo->pkt_hdr_size ==0)) ? 1 : 0; 2517 return is_dnx; 2518 } 2519 2520 /* is DPP or is DNX*/ 2521 static int 2522 device_is_sand(bkn_switch_info_t *sinfo) 2523 { 2524 int is_sand = 0; 2525 2526 if (device_is_dpp(sinfo) || device_is_dnx(sinfo)) { 2527 is_sand = 1; 2528 } 2529 return is_sand; 2530 } 2531 2532 static bkn_filter_t * 2533 bkn_match_rx_pkt(bkn_switch_info_t *sinfo, uint8_t *pkt, int pktlen, 2534 void *meta, int chan, bkn_filter_t *cbf) 2535 { 2536 struct list_head *list; 2537 bkn_filter_t *filter; 2538 kcom_filter_t scratch, *kf; 2539 uint8_t *oob = (uint8_t *)meta; 2540 int size, wsize; 2541 int idx, match; 2542 2543 list_for_each(list, &sinfo->rxpf_list) { 2544 filter = (bkn_filter_t *)list; 2545 kf = &filter->kf; 2546 memcpy(&scratch.data.b[0], 2547 &oob[kf->oob_data_offset], kf->oob_data_size); 2548 memcpy(&scratch.data.b[kf->oob_data_size], 2549 &pkt[kf->pkt_data_offset], kf->pkt_data_size); 2550 size = kf->oob_data_size + kf->pkt_data_size; 2551 wsize = BYTES2WORDS(size); 2552 DBG_VERB(("Filter: size = %d (%d), data = 0x%08x, mask = 0x%08x\n", 2553 size, wsize, kf->data.w[0], kf->mask.w[0])); 2554 2555 if (device_is_dnx(sinfo)) { 2556 DBG_DUNE(("Filter: size = %d (wsize %d)\n", size, wsize)); 2557 for (idx = 0; idx < wsize; idx++) 2558 { 2559 DBG_DUNE(("OOB[%d]: 0x%08x [0x%08x]\n", idx, kf->data.w[idx], kf->mask.w[idx])); 2560 } 2561 DBG_DUNE(("Meta Data [+ Selected Raw packet data]\n")); 2562 for (idx = 0; idx < wsize; idx++) 2563 { 2564 DBG_DUNE(("Scratch[%d]: 0x%08x\n", idx, scratch.data.w[idx])); 2565 } 2566 } 2567 2568 match = 1; 2569 if (match) { 2570 if (device_is_sand(sinfo)) 2571 { 2572 /** priority 0 means no priority check */ 2573 if (kf->priority && (kf->priority < (num_rx_prio * sinfo->rx_chans))) { 2574 if (kf->priority < (num_rx_prio * chan) || 2575 kf->priority >= (num_rx_prio * (chan + 1))) { 2576 match = 0; 2577 } 2578 } 2579 } 2580 else { 2581 if (kf->priority < (num_rx_prio * sinfo->rx_chans)) { 2582 if (kf->priority < (num_rx_prio * chan) || 2583 kf->priority >= (num_rx_prio * (chan + 1))) { 2584 match = 0; 2585 } 2586 } 2587 } 2588 } 2589 if (match) { 2590 for (idx = 0; idx < wsize; idx++) { 2591 scratch.data.w[idx] &= kf->mask.w[idx]; 2592 if (scratch.data.w[idx] != kf->data.w[idx]) { 2593 match = 0; 2594 break; 2595 } 2596 } 2597 } 2598 if (match) { 2599 if (kf->dest_type == KCOM_DEST_T_CB) { 2600 /* Check for custom filters */ 2601 if (knet_filter_cb != NULL && cbf != NULL) { 2602 memset(cbf, 0, sizeof(*cbf)); 2603 memcpy(&cbf->kf, kf, sizeof(cbf->kf)); 2604 if (knet_filter_cb(pkt, pktlen, sinfo->dev_no, 2605 meta, chan, &cbf->kf)) { 2606 filter->hits++; 2607 return cbf; 2608 } 2609 } else { 2610 DBG_FLTR(("Match, but not filter callback\n")); 2611 } 2612 } else { 2613 filter->hits++; 2614 return filter; 2615 } 2616 } 2617 } 2618 2619 return NULL; 2620 } 2621 2622 static bkn_priv_t * 2623 bkn_netif_lookup(bkn_switch_info_t *sinfo, int id) 2624 { 2625 struct list_head *list; 2626 bkn_priv_t *priv; 2627 int found; 2628 2629 /* Fast path */ 2630 if (id < sinfo->ndev_max) { 2631 if (sinfo->ndevs[id] != NULL) { 2632 DBG_NDEV(("Look up netif ID %d successful\n", id)); 2633 return netdev_priv(sinfo->ndevs[id]); 2634 } 2635 } 2636 2637 /* Slow path - should normally not get here */ 2638 found = 0; 2639 priv = NULL; 2640 list_for_each(list, &sinfo->ndev_list) { 2641 priv = (bkn_priv_t *)list; 2642 if (priv->id == id) { 2643 found = 1; 2644 break; 2645 } 2646 } 2647 if (found && priv != NULL) { 2648 return priv; 2649 } 2650 return NULL; 2651 } 2652 2653 static int 2654 bkn_hw_tstamp_rx_set(bkn_switch_info_t *sinfo, struct sk_buff *skb, uint32 *meta) 2655 { 2656 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 2657 uint32_t *md = meta; 2658 uint64_t ts = 0; 2659 2660 switch (sinfo->dcb_type) { 2661 case 26: 2662 case 32: 2663 case 33: 2664 ts = md[14]; 2665 ts = ts << 32 | md[12]; 2666 break; 2667 case 36: 2668 ts = md[10]; 2669 break; 2670 case 38: 2671 ts = md[4] & 0xffff; 2672 ts = ts << 32 | md[5]; 2673 break; 2674 default: 2675 return -1; 2676 } 2677 2678 if (knet_hw_tstamp_rx_time_upscale_cb) { 2679 if (knet_hw_tstamp_rx_time_upscale_cb(sinfo->dev_no, &ts) < 0) { 2680 return -1; 2681 } 2682 } 2683 2684 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 2685 shhwtstamps->hwtstamp = ns_to_ktime(be64_to_cpu(ts)); 2686 2687 return 0; 2688 } 2689 2690 static int 2691 bkn_add_rcpu_encap(bkn_switch_info_t *sinfo, struct sk_buff *skb, void *meta) 2692 { 2693 int pktlen = skb->len; 2694 uint32_t *dmeta, *smeta, wsize, psize; 2695 int idx; 2696 2697 /* Add and clear RCPU encapsulation */ 2698 if (sinfo->cmic_type == 'x') { 2699 psize = RCPU_HDR_SIZE + sinfo->pkt_hdr_size; 2700 skb_push(skb, psize); 2701 memset(skb->data, 0, RCPU_HDR_SIZE); 2702 } else { 2703 psize = RCPU_RX_ENCAP_SIZE; 2704 skb_push(skb, psize); 2705 memset(skb->data, 0, RCPU_RX_ENCAP_SIZE); 2706 } 2707 2708 /* RCPU Header */ 2709 memcpy(skb->data, &skb->data[psize], 12); 2710 if (rcpu_dmac != NULL) { 2711 memcpy(skb->data, bkn_rcpu_dmac, 6); 2712 } 2713 if (rcpu_smac != NULL) { 2714 memcpy(&skb->data[6], bkn_rcpu_smac, 6); 2715 } 2716 skb->data[12] = 0x81; 2717 skb->data[14] = rcpu_vlan >> 8; 2718 skb->data[15] = rcpu_vlan & 0xff; 2719 skb->data[16] = rcpu_ethertype >> 8; 2720 skb->data[17] = rcpu_ethertype & 0xff; 2721 skb->data[18] = sinfo->rcpu_sig >> 8; 2722 skb->data[19] = sinfo->rcpu_sig & 0xff; 2723 skb->data[20] = RCPU_OPCODE_RX; 2724 skb->data[21] = RCPU_F_MODHDR; 2725 skb->data[24] = pktlen >> 8; 2726 skb->data[25] = pktlen & 0xff; 2727 2728 /* Meta data */ 2729 dmeta = (uint32_t *)&skb->data[RCPU_HDR_SIZE]; 2730 smeta = sinfo->cmic_type == 'x' ? (uint32_t *)meta : (uint32_t *)meta + 2; 2731 wsize = sinfo->cmic_type == 'x' ? sinfo->pkt_hdr_size / 4 : sinfo->dcb_wsize - 3; 2732 for (idx = 0; idx < wsize; idx++) { 2733 dmeta[idx] = htonl(smeta[idx]); 2734 } 2735 2736 return 0; 2737 } 2738 2739 static void 2740 bkn_eth_type_update(struct sk_buff *skb, int ethertype) 2741 { 2742 #if defined(PM_ETH_TYPE) && defined(PM_FC_TYPE) 2743 /* Optionally override standard protocol */ 2744 skb->protocol = PM_ETH_TYPE; 2745 if (ethertype == ETH_P_FCOE) { 2746 skb->protocol = PM_FC_TYPE; 2747 } 2748 #endif 2749 } 2750 2751 #define BKN_DNX_BIT(x) (1<<(x)) 2752 #define BKN_DNX_RBIT(x) (~(1<<(x))) 2753 #ifdef __LITTLE_ENDIAN 2754 #define BKN_DNX_BYTE_SWAP(x) (x) 2755 #else 2756 #define BKN_DNX_BYTE_SWAP(x) ((((x) << 24)) | (((x) & 0xff00) << 8) | (((x) & 0xff0000) >> 8) | (((x) >> 24))) 2757 #endif 2758 2759 static int 2760 packet_is_untagged(uint16_t tpid) 2761 { 2762 int is_untagged = 0; 2763 2764 2765 /* 0x8100 is used in 802.1Q */ 2766 /* 0x8848 is used in 802.11ad, the dtag tpid can be set to anything besides 0x8848, 0x9100 is a typical value, but couldn't cover all scenarios. */ 2767 is_untagged = ((tpid != 0x8100) && (tpid != 0x8848) && (tpid != 0x9100)); 2768 return is_untagged; 2769 } 2770 2771 static void 2772 bkn_bitstream_set_field(uint32_t *input_buffer, uint32_t start_bit, uint32_t nof_bits, uint32_t field) 2773 { 2774 uint32_t place; 2775 uint32_t field_bit_i; 2776 uint32_t bit_indicator; 2777 2778 if( nof_bits > 32) 2779 { 2780 return; 2781 } 2782 2783 for( place=start_bit, field_bit_i = 0; field_bit_i< nof_bits; ++place, ++field_bit_i) 2784 { 2785 bit_indicator = field & BKN_DNX_BIT(nof_bits-field_bit_i-1); 2786 if(bit_indicator) 2787 { 2788 input_buffer[place>>5] |= (0x80000000 >> (place & 0x0000001F)); 2789 } 2790 else 2791 { 2792 input_buffer[place>>5] &= ~(0x80000000 >> (place & 0x0000001F)); 2793 } 2794 } 2795 return; 2796 } 2797 2798 static void 2799 bkn_bitstream_get_field(uint8_t *input_buffer, uint32_t start_bit, uint32_t nof_bits, uint32_t *output_value) 2800 { 2801 uint32_t idx; 2802 uint32_t buf_sizes=0; 2803 uint32_t tmp_output_value[2]={0}; 2804 uint32_t first_byte_ndx; 2805 uint32_t last_byte_ndx; 2806 uint32_t place; 2807 uint32_t field_bit_i; 2808 uint8_t *tmp_output_value_u8_ptr = (uint8_t*)&tmp_output_value; 2809 uint32_t bit_indicator; 2810 2811 if (nof_bits > 32) 2812 { 2813 return; 2814 } 2815 2816 first_byte_ndx = start_bit / 8; 2817 last_byte_ndx = ((start_bit + nof_bits - 1) / 8); 2818 *output_value=0; 2819 2820 /* get 32 bit value, MSB */ 2821 for (idx = first_byte_ndx; idx <= last_byte_ndx; ++idx) 2822 { 2823 tmp_output_value_u8_ptr[last_byte_ndx - idx] = input_buffer[idx]; 2824 buf_sizes += 8; 2825 } 2826 tmp_output_value[0] = BKN_DNX_BYTE_SWAP(tmp_output_value[0]); 2827 if (last_byte_ndx > 4) 2828 { 2829 tmp_output_value[1] = BKN_DNX_BYTE_SWAP(tmp_output_value[1]); 2830 } 2831 2832 place = buf_sizes - (start_bit % 8 + nof_bits); 2833 for (field_bit_i = 0; field_bit_i< nof_bits; ++place, ++field_bit_i) 2834 { 2835 uint32_t result; 2836 result = tmp_output_value[place>>5] & BKN_DNX_BIT(place & 0x0000001F); 2837 if (result) 2838 { 2839 bit_indicator = 1; 2840 } else { 2841 bit_indicator = 0; 2842 } 2843 *output_value |= bit_indicator << field_bit_i; 2844 } 2845 return; 2846 } 2847 2848 static void 2849 bkn_dpp_packet_parse_ftmh(bkn_switch_info_t *sinfo, uint8_t hdr_buff[], bkn_dune_system_header_info_t *packet_info) 2850 { 2851 uint32_t header_ptr = 0; 2852 uint32_t dsp_ext_exist=0; 2853 uint32_t fld_val; 2854 2855 header_ptr = packet_info->ntwrk_header_ptr; 2856 2857 /* Packet-size */ 2858 bkn_bitstream_get_field( 2859 &hdr_buff[header_ptr], 2860 BKN_DPP_FTMH_PKT_SIZE_MSB, 2861 BKN_DPP_FTMH_PKT_SIZE_NOF_BITS, 2862 &fld_val); 2863 packet_info->ftmh.packet_size = fld_val; 2864 /* Traffic-class */ 2865 bkn_bitstream_get_field( 2866 &hdr_buff[header_ptr], 2867 BKN_DPP_FTMH_TC_MSB, 2868 BKN_DPP_FTMH_TC_NOF_BITS, 2869 &fld_val); 2870 packet_info->ftmh.prio = fld_val; 2871 /* Source-system-port-aggregate */ 2872 bkn_bitstream_get_field( 2873 &hdr_buff[header_ptr], 2874 BKN_DPP_FTMH_SRC_SYS_PORT_MSB, 2875 BKN_DPP_FTMH_SRC_SYS_PORT_NOF_BITS, 2876 &fld_val); 2877 packet_info->ftmh.src_sys_port = fld_val; 2878 /* TM-action-type */ 2879 bkn_bitstream_get_field( 2880 &hdr_buff[header_ptr], 2881 BKN_DPP_FTMH_ACTION_TYPE_MSB, 2882 BKN_DPP_FTMH_ACTION_TYPE_NOF_BITS, 2883 &fld_val); 2884 packet_info->ftmh.action_type = fld_val; 2885 /* PPH-type */ 2886 bkn_bitstream_get_field( 2887 &hdr_buff[header_ptr], 2888 BKN_DPP_FTMH_PPH_TYPE_MSB, 2889 BKN_DPP_FTMH_PPH_TYPE_NOF_BITS, 2890 &fld_val); 2891 packet_info->ftmh.pph_type = fld_val; 2892 2893 packet_info->ntwrk_header_ptr += BKN_DPP_FTMH_SIZE_BYTE; 2894 DBG_DUNE(("FTMH(%d) Packet-size %d Action-type %d PPH-type %d Source-system-port 0x%x Traffic-class %d\n", 2895 packet_info->ntwrk_header_ptr, packet_info->ftmh.packet_size, packet_info->ftmh.action_type, 2896 packet_info->ftmh.pph_type, packet_info->ftmh.src_sys_port, packet_info->ftmh.prio)); 2897 2898 /* LB-Key ext */ 2899 if ((sinfo->pkt_hdr_size & BKN_DPP_FTMH_LB_EXT_EN) == BKN_DPP_FTMH_LB_EXT_EN) 2900 { 2901 packet_info->ntwrk_header_ptr += BKN_DPP_FTMH_LB_EXT_SIZE_BYTE; 2902 DBG_DUNE(("FTMH(%d) FTMH LB-Key Extension is present\n", packet_info->ntwrk_header_ptr)); 2903 } 2904 /* DSP ext*/ 2905 fld_val = 0; 2906 bkn_bitstream_get_field( 2907 &hdr_buff[header_ptr], 2908 BKN_DPP_FTMH_EXT_DSP_EXIST_MSB, 2909 BKN_DPP_FTMH_EXT_DSP_EXIST_NOF_BITS, 2910 &dsp_ext_exist); 2911 if (dsp_ext_exist) 2912 { 2913 packet_info->ntwrk_header_ptr += BKN_DPP_FTMH_DEST_EXT_SIZE_BYTE; 2914 DBG_DUNE(("FTMH(%d) DSP-extension-present 1\n", packet_info->ntwrk_header_ptr)); 2915 } 2916 2917 /* stacking ext */ 2918 if ((sinfo->pkt_hdr_size & BKN_DPP_FTMH_STACKING_EXT_EN) == BKN_DPP_FTMH_STACKING_EXT_EN) 2919 { 2920 packet_info->ntwrk_header_ptr += BKN_DPP_FTMH_STACKING_SIZE_BYTE; 2921 DBG_DUNE(("FTMH(%d) FTMH Stacking Extension is present\n", packet_info->ntwrk_header_ptr)); 2922 } 2923 return; 2924 } 2925 2926 static void 2927 bkn_dpp_packet_parse_internal(bkn_switch_info_t *sinfo, uint8_t hdr_buff[], bkn_dune_system_header_info_t *packet_info) 2928 { 2929 uint32_t header_ptr = 0; 2930 uint32_t fld_val; 2931 uint32_t eei_extension_present = 0; 2932 uint32_t learn_extension_present = 0; 2933 uint32_t fhei_size = 0; 2934 uint32_t forward_code; 2935 uint8_t is_trapped = 0; 2936 2937 header_ptr = packet_info->ntwrk_header_ptr; 2938 2939 bkn_bitstream_get_field( 2940 &hdr_buff[header_ptr], 2941 BKN_DPP_PPH_EEI_EXTENSION_PRESENT_MSB, 2942 BKN_DPP_PPH_EEI_EXTENSION_PRESENT_NOF_BITS, 2943 &eei_extension_present); 2944 bkn_bitstream_get_field( 2945 &hdr_buff[header_ptr], 2946 BKN_DPP_PPH_LEARN_EXENSION_PRESENT_MSB, 2947 BKN_DPP_PPH_LEARN_EXENSION_PRESENT_NOF_BITS, 2948 &learn_extension_present); 2949 bkn_bitstream_get_field( 2950 &hdr_buff[header_ptr], 2951 BKN_DPP_PPH_FHEI_SIZE_MSB, 2952 BKN_DPP_PPH_FHEI_SIZE_NOF_BITS, 2953 &fhei_size); 2954 bkn_bitstream_get_field( 2955 &hdr_buff[header_ptr], 2956 BKN_DPP_PPH_FORWARD_CODE_MSB, 2957 BKN_DPP_PPH_FORWARD_CODE_NOF_BITS, 2958 &forward_code); 2959 /* 7: CPU-Trap */ 2960 is_trapped = (uint8_t)(forward_code == 7); 2961 2962 bkn_bitstream_get_field( 2963 &hdr_buff[header_ptr], 2964 BKN_DPP_PPH_VSI_MSB, 2965 BKN_DPP_PPH_VSI_NOF_BITS, 2966 &fld_val); 2967 packet_info->internal.vsi = fld_val; 2968 2969 /* size of PPH base is 7 */ 2970 packet_info->ntwrk_header_ptr += BKN_DPP_PPH_SIZE_BYTE; 2971 header_ptr = packet_info->ntwrk_header_ptr; 2972 2973 DBG_DUNE(("PPH(%d) Forward-Code %d EEI-Extension %d Learn-Extension %d VSI %d FHEI-size %d\n", packet_info->ntwrk_header_ptr, 2974 forward_code, eei_extension_present, learn_extension_present, packet_info->internal.vsi, fhei_size)); 2975 2976 /* PPH extension */ 2977 if (is_trapped && (fhei_size == 1)) 2978 { 2979 /* CPU trap code qualifier */ 2980 bkn_bitstream_get_field( 2981 &hdr_buff[header_ptr], 2982 BKN_DPP_PPH_FHEI_TRAP_SNOOP_3B_CPU_TRAP_CODE_QUALIFIER_MSB, 2983 BKN_DPP_PPH_FHEI_TRAP_SNOOP_3B_CPU_TRAP_CODE_QUALIFIER_NOF_BITS, 2984 &fld_val); 2985 packet_info->internal.trap_qualifier = fld_val; 2986 /* CPU trap code */ 2987 bkn_bitstream_get_field( 2988 &hdr_buff[header_ptr], 2989 BKN_DPP_PPH_FHEI_TRAP_SNOOP_3B_CPU_TRAP_CODE_MSB, 2990 BKN_DPP_PPH_FHEI_TRAP_SNOOP_3B_CPU_TRAP_CODE_NOF_BITS, 2991 &fld_val); 2992 packet_info->internal.trap_id = fld_val; 2993 } 2994 switch(fhei_size) { 2995 case 1: 2996 packet_info->ntwrk_header_ptr += BKN_DPP_PPH_FHEI_3B_SIZE_BYTE; 2997 break; 2998 case 2: 2999 packet_info->ntwrk_header_ptr += BKN_DPP_PPH_FHEI_5B_SIZE_BYTE; 3000 break; 3001 case 3: 3002 packet_info->ntwrk_header_ptr += BKN_DPP_PPH_FHEI_8B_SIZE_BYTE; 3003 break; 3004 default: 3005 break; 3006 } 3007 if (eei_extension_present) { 3008 packet_info->ntwrk_header_ptr += BKN_DPP_PPH_EXPLICIT_EDITING_INFOMATION_EXTENSION_SIZE_BYTE; 3009 } 3010 if (learn_extension_present) { 3011 packet_info->ntwrk_header_ptr += BKN_DPP_PPH_LEARN_EXTENSION_SIZE_BYTE; 3012 } 3013 3014 DBG_DUNE(("FHEI(%d) trap_qualifier 0x%x trap_id 0x%x\n", packet_info->ntwrk_header_ptr, packet_info->internal.trap_qualifier, packet_info->internal.trap_id)); 3015 return; 3016 } 3017 3018 static int 3019 bkn_dpp_packet_header_parse(bkn_switch_info_t *sinfo, uint8 *buff, uint32_t buff_len, bkn_dune_system_header_info_t *packet_info) 3020 { 3021 uint8_t hdr_buff[BKN_DPP_HDR_MAX_SIZE]; 3022 uint32_t hdr_size; 3023 uint8_t has_internal = 0; 3024 3025 if ((buff == NULL) || (packet_info == NULL)) { 3026 return -1; 3027 } 3028 hdr_size = buff_len < BKN_DPP_HDR_MAX_SIZE ? buff_len: BKN_DPP_HDR_MAX_SIZE; 3029 memcpy(hdr_buff, buff, hdr_size); 3030 3031 /* FTMH */ 3032 bkn_dpp_packet_parse_ftmh(sinfo, hdr_buff, packet_info); 3033 if (packet_info->ftmh.packet_size != (buff_len + 2)) { 3034 DBG_DUNE(("FTMH packet size verfication failed, %d-%d\n", packet_info->ftmh.packet_size, buff_len)); 3035 memset(packet_info, 0, sizeof(bkn_dune_system_header_info_t)); 3036 return -1; 3037 } 3038 switch (packet_info->ftmh.pph_type) { 3039 case 0: 3040 has_internal = 0; 3041 break; 3042 case 1: 3043 has_internal = 1; 3044 break; 3045 case 2: /* PPH OAM-TS only */ 3046 case 3: /* PPH Base + OAM-TS */ 3047 /* OTSH immediately follows the FTMH when present */ 3048 packet_info->ntwrk_header_ptr += 6; 3049 DBG_DUNE(("FTMH + OAM-TS(%d)\n", packet_info->ntwrk_header_ptr)); 3050 has_internal = 1; 3051 break; 3052 default: 3053 break; 3054 } 3055 3056 if (has_internal) { 3057 bkn_dpp_packet_parse_internal(sinfo, &hdr_buff[0], packet_info); 3058 } 3059 3060 3061 /* ignore packets with a double set of FTMH,internals */ 3062 /* ignore the user header size */ 3063 return 0; 3064 } 3065 3066 static int 3067 bkn_dnx_packet_header_parse(bkn_switch_info_t *sinfo, uint8 *buf, uint32_t buf_len, bkn_dune_system_header_info_t *packet_info) 3068 { 3069 uint32_t fld_val; 3070 uint32_t hdr_size = 0; 3071 uint32_t pkt_offset_ingress_untrapped =0; 3072 uint8_t tm_dst_ext_present = 0; 3073 uint8_t app_specific_ext_size = 0; 3074 uint8_t flow_id_ext_size = 0; 3075 uint8_t bier_bfr_ext_size = 0; 3076 uint8_t is_pph_en = 0; 3077 uint8_t is_tsh_en = 0; 3078 3079 if ((buf == NULL) || (packet_info == NULL)) { 3080 return -1; 3081 } 3082 3083 /* FTMH: Source-System-Port-Aggregate */ 3084 bkn_bitstream_get_field( 3085 &buf[hdr_size], 3086 BKN_DNX_FTMH_SRC_SYS_PORT_AGGREGATE_MSB, 3087 BKN_DNX_FTMH_SRC_SYS_PORT_AGGREGATE_NOF_BITS, 3088 &fld_val); 3089 packet_info->ftmh_spa = fld_val; 3090 /* FTMH: PPH-Type TSH */ 3091 bkn_bitstream_get_field( 3092 &buf[hdr_size], 3093 BKN_DNX_FTMH_PPH_TYPE_IS_TSH_EN_MSB, 3094 BKN_DNX_FTMH_PPH_TYPE_IS_TSH_EN_NOF_BITS, 3095 &fld_val); 3096 is_tsh_en = fld_val; 3097 /* FTMH: PPH-Type PPH base */ 3098 bkn_bitstream_get_field( 3099 &buf[hdr_size], 3100 BKN_DNX_FTMH_PPH_TYPE_IS_PPH_EN_MSB, 3101 BKN_DNX_FTMH_PPH_TYPE_IS_PPH_EN_NOF_BITS, 3102 &fld_val); 3103 is_pph_en = fld_val; 3104 /* FTMH: TM-Destination-Extension-Present */ 3105 bkn_bitstream_get_field( 3106 &buf[hdr_size], 3107 BKN_DNX_FTMH_TM_DST_EXT_PRESENT_MSB, 3108 BKN_DNX_FTMH_TM_DST_EXT_PRESENT_NOF_BITS, 3109 &fld_val); 3110 tm_dst_ext_present = fld_val; 3111 /* FTMH: Application-Specific-Extension-Size */ 3112 bkn_bitstream_get_field( 3113 &buf[hdr_size], 3114 BKN_DNX_FTMH_APP_SPECIFIC_EXT_SIZE_MSB, 3115 BKN_DNX_FTMH_APP_SPECIFIC_EXT_SIZE_NOF_BITS, 3116 &fld_val); 3117 app_specific_ext_size = fld_val; 3118 /* FTMH: Flow-ID-Extension-Size */ 3119 bkn_bitstream_get_field( 3120 &buf[hdr_size], 3121 BKN_DNX_FTMH_FLOW_ID_EXT_SIZE_MSB, 3122 BKN_DNX_FTMH_FLOW_ID_EXT_SIZE_NOF_BITS, 3123 &fld_val); 3124 flow_id_ext_size = fld_val; 3125 /* FTMH: BIER-BFR-Extension-Size */ 3126 bkn_bitstream_get_field( 3127 &buf[hdr_size], 3128 BKN_DNX_FTMH_BIER_BFR_EXT_SIZE_MSB, 3129 BKN_DNX_FTMH_BIER_BFR_EXT_SIZE_NOF_BITS, 3130 &fld_val); 3131 bier_bfr_ext_size = fld_val; 3132 3133 hdr_size = BKN_DNX_FTMH_BASE_SIZE; 3134 pkt_offset_ingress_untrapped = BKN_DNX_FTMH_BASE_SIZE; 3135 3136 DBG_DUNE(("FTMH(%d) source-system-port 0x%x is_tsh_en %d is_pph_en %d\n", 3137 hdr_size, packet_info->ftmh_spa, is_tsh_en, is_pph_en)); 3138 3139 /* FTMH LB-Key Extension */ 3140 if (sinfo->ftmh_lb_key_ext_size > 0) 3141 { 3142 hdr_size += sinfo->ftmh_lb_key_ext_size; 3143 DBG_DUNE(("FTMH LB-Key Extension(%d) is present\n", sinfo->ftmh_lb_key_ext_size)); 3144 } 3145 /* FTMH Stacking Extension */ 3146 if (sinfo->ftmh_stacking_ext_size > 0) 3147 { 3148 hdr_size += sinfo->ftmh_stacking_ext_size; 3149 DBG_DUNE(("FTMH Stacking Extension(%d) is present\n", sinfo->ftmh_stacking_ext_size)); 3150 } 3151 /* FTMH BIER BFR Extension */ 3152 if (bier_bfr_ext_size > 0) 3153 { 3154 hdr_size += BKN_DNX_FTMH_BIER_BFR_EXT_SIZE; 3155 DBG_DUNE(("FTMH BIER BFR Extension(2) is present\n")); 3156 } 3157 /* FTMH TM Destination Extension */ 3158 if (tm_dst_ext_present > 0) 3159 { 3160 hdr_size += BKN_DNX_FTMH_TM_DST_EXT_SIZE; 3161 DBG_DUNE(("FTMH TM Destination Extension(3) is present\n")); 3162 } 3163 /* FTMH Application Specific Extension */ 3164 if (app_specific_ext_size > 0) 3165 { 3166 hdr_size += BKN_DNX_FTMH_APP_SPECIFIC_EXT_SIZE; 3167 DBG_DUNE(("FTMH Application Specific Extension(6) is present\n")); 3168 } 3169 /* FTMH Flow-ID Extension */ 3170 if (flow_id_ext_size > 0) 3171 { 3172 hdr_size += BKN_DNX_FTMH_FLOW_ID_EXT_SIZE; 3173 DBG_DUNE(("FTMH Flow-ID Extension(3) is present\n")); 3174 } 3175 3176 /* Given the packet is trapped to CPU */ 3177 3178 /* Time-Stamp Header */ 3179 if (is_tsh_en == TRUE) 3180 { 3181 hdr_size += BKN_DNX_TSH_SIZE; 3182 DBG_DUNE(("Time-Stamp Header(4) is present\n")); 3183 } 3184 3185 /* Packet Processing Header */ 3186 if (is_pph_en) 3187 { 3188 uint8_t learn_ext_present; 3189 uint8_t fhei_size; 3190 uint8_t lif_ext_type; 3191 3192 switch (sinfo->pph_base_size) 3193 { 3194 case BKN_DNX_PPH_BASE_TYPE_9: 3195 /* FTMH: Forward-Domain */ 3196 bkn_bitstream_get_field( 3197 &buf[hdr_size], 3198 BKN_DNX_PPH_9_FORWARD_DOMAIN_MSB, 3199 BKN_DNX_PPH_9_FORWARD_DOMAIN_NOF_BITS, 3200 &fld_val); 3201 packet_info->pph_forward_domain = fld_val; 3202 /* FTMH: Learn-Extension-Present */ 3203 bkn_bitstream_get_field( 3204 &buf[hdr_size], 3205 BKN_DNX_PPH_9_LEARN_EXT_PRESENT_MSB, 3206 BKN_DNX_PPH_9_LEARN_EXT_PRESENT_NOF_BITS, 3207 &fld_val); 3208 learn_ext_present = fld_val; 3209 /* FTMH: FHEI-Size */ 3210 bkn_bitstream_get_field( 3211 &buf[hdr_size], 3212 BKN_DNX_PPH_9_FHEI_SIZE_MSB, 3213 BKN_DNX_PPH_9_FHEI_SIZE_NOF_BITS, 3214 &fld_val); 3215 fhei_size = fld_val; 3216 /* FTMH: LIF-Extension-Type */ 3217 bkn_bitstream_get_field( 3218 &buf[hdr_size], 3219 BKN_DNX_PPH_9_LIF_EXT_TYPE_MSB, 3220 BKN_DNX_PPH_9_LIF_EXT_TYPE_NOF_BITS, 3221 &fld_val); 3222 lif_ext_type = fld_val; 3223 3224 hdr_size += BKN_DNX_PPH_BASE_TYPE_9; 3225 DBG_DUNE(("PPH(10) FWD_DOMAIN %d, LEARN_EXT %d, FHEI_SIZE %d, LIF_EXT %d \n", 3226 packet_info->pph_forward_domain, learn_ext_present, fhei_size, lif_ext_type)); 3227 break; 3228 case BKN_DNX_PPH_BASE_TYPE_10: 3229 /* FTMH: Forward-Domain */ 3230 bkn_bitstream_get_field( 3231 &buf[hdr_size], 3232 BKN_DNX_PPH_10_FORWARD_DOMAIN_MSB, 3233 BKN_DNX_PPH_10_FORWARD_DOMAIN_NOF_BITS, 3234 &fld_val); 3235 packet_info->pph_forward_domain = fld_val; 3236 /* FTMH: Learn-Extension-Present */ 3237 bkn_bitstream_get_field( 3238 &buf[hdr_size], 3239 BKN_DNX_PPH_10_LEARN_EXT_PRESENT_MSB, 3240 BKN_DNX_PPH_10_LEARN_EXT_PRESENT_NOF_BITS, 3241 &fld_val); 3242 learn_ext_present = fld_val; 3243 /* FTMH: FHEI-Size */ 3244 bkn_bitstream_get_field( 3245 &buf[hdr_size], 3246 BKN_DNX_PPH_10_FHEI_SIZE_MSB, 3247 BKN_DNX_PPH_10_FHEI_SIZE_NOF_BITS, 3248 &fld_val); 3249 fhei_size = fld_val; 3250 /* FTMH: LIF-Extension-Type */ 3251 bkn_bitstream_get_field( 3252 &buf[hdr_size], 3253 BKN_DNX_PPH_10_LIF_EXT_TYPE_MSB, 3254 BKN_DNX_PPH_10_LIF_EXT_TYPE_NOF_BITS, 3255 &fld_val); 3256 lif_ext_type = fld_val; 3257 3258 hdr_size += BKN_DNX_PPH_BASE_TYPE_10; 3259 DBG_DUNE(("PPH(10) FWD_DOMAIN %d, LEARN_EXT %d, FHEI_SIZE %d, LIF_EXT %d \n", 3260 packet_info->pph_forward_domain, learn_ext_present, fhei_size, lif_ext_type)); 3261 break; 3262 case BKN_DNX_PPH_BASE_TYPE_12: 3263 /* FTMH: Forward-Domain */ 3264 bkn_bitstream_get_field( 3265 &buf[hdr_size], 3266 BKN_DNX_PPH_12_FORWARD_DOMAIN_MSB, 3267 BKN_DNX_PPH_12_FORWARD_DOMAIN_NOF_BITS, 3268 &fld_val); 3269 packet_info->pph_forward_domain = fld_val; 3270 /* FTMH: Learn-Extension-Present */ 3271 bkn_bitstream_get_field( 3272 &buf[hdr_size], 3273 BKN_DNX_PPH_12_LEARN_EXT_PRESENT_MSB, 3274 BKN_DNX_PPH_12_LEARN_EXT_PRESENT_NOF_BITS, 3275 &fld_val); 3276 learn_ext_present = fld_val; 3277 /* FTMH: FHEI-Size */ 3278 bkn_bitstream_get_field( 3279 &buf[hdr_size], 3280 BKN_DNX_PPH_12_FHEI_SIZE_MSB, 3281 BKN_DNX_PPH_12_FHEI_SIZE_NOF_BITS, 3282 &fld_val); 3283 fhei_size = fld_val; 3284 /* FTMH: LIF-Extension-Type */ 3285 bkn_bitstream_get_field( 3286 &buf[hdr_size], 3287 BKN_DNX_PPH_12_LIF_EXT_TYPE_MSB, 3288 BKN_DNX_PPH_12_LIF_EXT_TYPE_NOF_BITS, 3289 &fld_val); 3290 lif_ext_type = fld_val; 3291 3292 hdr_size += BKN_DNX_PPH_BASE_TYPE_12; 3293 DBG_DUNE(("PPH(12) FWD_DOMAIN %d, LEARN_EXT %d, FHEI_SIZE %d, LIF_EXT %d \n", 3294 packet_info->pph_forward_domain, learn_ext_present, fhei_size, lif_ext_type)); 3295 break; 3296 default: 3297 fhei_size = 0; 3298 lif_ext_type = 0; 3299 learn_ext_present = 0; 3300 break; 3301 } 3302 if (fhei_size) 3303 { 3304 switch (fhei_size) 3305 { 3306 case BKN_DNX_PPH_FHEI_TYPE_SZ0: 3307 hdr_size += BKN_DNX_PPH_FHEI_SZ0_SIZE; 3308 DBG_DUNE(("FHEI(3) is present\n")); 3309 break; 3310 case BKN_DNX_PPH_FHEI_TYPE_SZ1: 3311 /* FHEI: Code */ 3312 bkn_bitstream_get_field( 3313 &buf[hdr_size], 3314 BKN_DNX_PPH_FHEI_TRAP_5B_TYPE_MSB, 3315 BKN_DNX_PPH_FHEI_TRAP_5B_TYPE_NOF_BITS, 3316 &fld_val); 3317 packet_info->fhei_type = fld_val; 3318 /* FHEI-Size == 5B, FHEI-Type == Trap/Sniff */ 3319 if (packet_info->fhei_type == 0x5) 3320 { 3321 /* FHEI: Qualifier */ 3322 bkn_bitstream_get_field( 3323 &buf[hdr_size], 3324 BKN_DNX_PPH_FHEI_TRAP_5B_QUALIFIER_MSB, 3325 BKN_DNX_PPH_FHEI_TRAP_5B_QUALIFIER_NOF_BITS, 3326 &fld_val); 3327 packet_info->fhei_qualifier = fld_val; 3328 /* FHEI: Code */ 3329 bkn_bitstream_get_field( 3330 &buf[hdr_size], 3331 BKN_DNX_PPH_FHEI_TRAP_5B_CODE_MSB, 3332 BKN_DNX_PPH_FHEI_TRAP_5B_CODE_NOF_BITS, 3333 &fld_val); 3334 packet_info->fhei_code = fld_val; 3335 } 3336 hdr_size += BKN_DNX_PPH_FHEI_SZ1_SIZE; 3337 DBG_DUNE(("FHEI(5) is present code 0x%x qualifier 0x%x\n", packet_info->fhei_code, packet_info->fhei_qualifier)); 3338 break; 3339 case BKN_DNX_PPH_FHEI_TYPE_SZ2: 3340 hdr_size += BKN_DNX_PPH_FHEI_SZ1_SIZE; 3341 DBG_DUNE(("FHEI(8) is present\n")); 3342 break; 3343 } 3344 } 3345 3346 /* PPH LIF Extension */ 3347 if (lif_ext_type) 3348 { 3349 hdr_size += sinfo->pph_lif_ext_size[lif_ext_type]; 3350 DBG_DUNE(("PPH LIF Extension(%d) is present\n", sinfo->pph_lif_ext_size[lif_ext_type])); 3351 } 3352 3353 /* PPH Learn Extension */ 3354 if (learn_ext_present) 3355 { 3356 hdr_size += BKN_DNX_PPH_LEARN_EXT_SIZE; 3357 DBG_DUNE(("PPH Learn Extension(19) is present\n")); 3358 } 3359 } 3360 3361 /* UDH Header */ 3362 if (sinfo->udh_enable) 3363 { 3364 uint8 data_type_0; 3365 uint8 data_type_1; 3366 uint8 data_type_2; 3367 uint8 data_type_3; 3368 3369 DBG_DUNE(("UDH base(1) is present\n")); 3370 3371 /* UDH: UDH-Data-Type[0] */ 3372 bkn_bitstream_get_field( 3373 &buf[hdr_size], 3374 BKN_DNX_UDH_DATA_TYPE_0_MSB, 3375 BKN_DNX_UDH_DATA_TYPE_0_NOF_BITS, 3376 &fld_val); 3377 data_type_0 = fld_val; 3378 hdr_size += sinfo->udh_length_type[data_type_0]; 3379 /* UDH: UDH-Data-Type[1] */ 3380 bkn_bitstream_get_field( 3381 &buf[hdr_size], 3382 BKN_DNX_UDH_DATA_TYPE_1_MSB, 3383 BKN_DNX_UDH_DATA_TYPE_1_NOF_BITS, 3384 &fld_val); 3385 data_type_1 = fld_val; 3386 hdr_size += sinfo->udh_length_type[data_type_1]; 3387 /* UDH: UDH-Data-Type[2] */ 3388 bkn_bitstream_get_field( 3389 &buf[hdr_size], 3390 BKN_DNX_UDH_DATA_TYPE_2_MSB, 3391 BKN_DNX_UDH_DATA_TYPE_2_NOF_BITS, 3392 &fld_val); 3393 data_type_2 = fld_val; 3394 hdr_size += sinfo->udh_length_type[data_type_2]; 3395 /* UDH: UDH-Data-Type[3] */ 3396 bkn_bitstream_get_field( 3397 &buf[hdr_size], 3398 BKN_DNX_UDH_DATA_TYPE_3_MSB, 3399 BKN_DNX_UDH_DATA_TYPE_3_NOF_BITS, 3400 &fld_val); 3401 data_type_3 = fld_val; 3402 hdr_size += sinfo->udh_length_type[data_type_3]; 3403 hdr_size += BKN_DNX_UDH_BASE_SIZE; 3404 } 3405 3406 /* 3407 * Check if fhei_type if of type trap 3408 * There is a RISK that raw packet data is parsed and fhei_type is 0x5 by chance 3409 */ 3410 if (packet_info->fhei_type == 0x5) 3411 { 3412 /* Done for ingress trapped packets */ 3413 packet_info->system_header_size = hdr_size; 3414 DBG_DUNE(("Total Size of Headers is %d\n", packet_info->system_header_size)); 3415 return 0; 3416 } 3417 else 3418 { 3419 /* New generated header will be FTMH(80b), TSH(32b), PPH(96b), FHEI(40b) */ 3420 3421 /* Revert packet_info except info from FTHM base header */ 3422 packet_info->fhei_qualifier = 0; 3423 packet_info->fhei_code = 0; 3424 packet_info->fhei_type = 0; 3425 hdr_size = pkt_offset_ingress_untrapped; 3426 3427 /* Time-Stamp Header */ 3428 hdr_size += BKN_DNX_TSH_SIZE; 3429 DBG_DUNE(("Time-Stamp Header(4) is present\n")); 3430 3431 /** Packet Processing Header */ 3432 bkn_bitstream_get_field( 3433 &buf[hdr_size], 3434 BKN_DNX_PPH_12_FORWARD_DOMAIN_MSB, 3435 BKN_DNX_PPH_12_FORWARD_DOMAIN_NOF_BITS, 3436 &fld_val); 3437 packet_info->pph_forward_domain = fld_val; 3438 hdr_size += BKN_DNX_PPH_BASE_TYPE_12; 3439 DBG_DUNE(("PPH(12) is present\n")); 3440 3441 /* 3442 * FHEI(Trap,40) 3443 * 5B-FHEI format for sys_hdr_generation_profile:J2-OAM 3444 * 8b' 0 3445 * 19b'oam_id 3446 * 9b' cpu_trap_code: INGRESS_TRAP_ID is always 0 here 3447 * 4b' type(0x5): J2-Configuration FHEI Type for CPU trap 3448 */ 3449 hdr_size += BKN_DNX_PPH_FHEI_SZ1_SIZE; 3450 DBG_DUNE(("FHEI(5) is present\n")); 3451 } 3452 packet_info->system_header_size = hdr_size; 3453 DBG_DUNE(("Total Size of Headers is %d\n", packet_info->system_header_size)); 3454 return 0; 3455 } 3456 3457 3458 static int 3459 bkn_do_api_rx(bkn_switch_info_t *sinfo, int chan, int budget) 3460 { 3461 bkn_priv_t *priv; 3462 bkn_dcb_chain_t *dcb_chain; 3463 struct sk_buff *skb; 3464 bkn_filter_t cbf; 3465 bkn_filter_t *filter; 3466 uint32_t err_woff; 3467 uint32_t *dcb, *meta; 3468 uint8_t *pkt; 3469 uint64_t pkt_dma; 3470 int drop_api; 3471 int ethertype; 3472 int pktlen; 3473 int idx; 3474 int dcbs_done = 0; 3475 bkn_dune_system_header_info_t packet_info = {0}; 3476 uint32_t dnx_meta_data[3] = {0}; 3477 3478 dcb_chain = sinfo->rx[chan].api_dcb_chain; 3479 if (dcb_chain == NULL) { 3480 /* No active chains */ 3481 return 0; 3482 } 3483 3484 while (dcb_chain->dcb_cur < dcb_chain->dcb_cnt) { 3485 dcb = &dcb_chain->dcb_mem[dcb_chain->dcb_cur * sinfo->dcb_wsize]; 3486 DBG_VERB(("DCB %2d: 0x%08x\n", 3487 dcb_chain->dcb_cur, dcb[sinfo->dcb_wsize-1])); 3488 if ((dcb[sinfo->dcb_wsize-1] & (1 << 31)) == 0) { 3489 break; 3490 } 3491 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 3492 /* Handle for Continuous DMA mode */ 3493 if (dcbs_done >= budget) { 3494 break; 3495 } 3496 if ((sinfo->cmic_type == 'x' && dcb[2] & (1 << 18)) || 3497 (sinfo->cmic_type != 'x' && dcb[1] & (1 << 18))) { 3498 dcb[sinfo->dcb_wsize-1] |= SOC_DCB_KNET_DONE; 3499 bkn_api_rx_chain_done(sinfo, chan); 3500 dcb_chain = sinfo->rx[chan].api_dcb_chain; 3501 if (dcb_chain == NULL) { 3502 break; 3503 } 3504 continue; 3505 } 3506 } 3507 if ((sinfo->cmic_type == 'x' && (dcb[2] & (1 << 16)) == 0) || 3508 (sinfo->cmic_type != 'x' && (dcb[1] & (1 << 16)) == 0)) { 3509 sinfo->rx[chan].chain_complete = 1; 3510 } 3511 sinfo->rx[chan].pkts++; 3512 if (sinfo->cmic_type == 'x') { 3513 pkt_dma = BUS_TO_DMA_HI(dcb[1]); 3514 pkt_dma = pkt_dma << 32 | dcb[0]; 3515 } else { 3516 pkt_dma = dcb[0]; 3517 } 3518 pkt = (uint8_t *)kernel_bde->p2l(sinfo->dev_no, (sal_paddr_t)pkt_dma); 3519 if (sinfo->cmic_type == 'x') { 3520 if (device_is_dnx(sinfo)){ 3521 meta = &dnx_meta_data[0]; 3522 /* get error bit from last DCB words */ 3523 err_woff = 2; 3524 meta[err_woff] = dcb[sinfo->dcb_wsize-1]; 3525 } else { 3526 meta = (uint32_t *)pkt; 3527 err_woff = sinfo->pkt_hdr_size / sizeof(uint32_t) - 1; 3528 meta[err_woff] = dcb[sinfo->dcb_wsize-1]; 3529 } 3530 } else { 3531 meta = dcb; 3532 err_woff = sinfo->dcb_wsize - 1; 3533 } 3534 pktlen = dcb[sinfo->dcb_wsize-1] & SOC_DCB_KNET_COUNT_MASK; 3535 bkn_dump_pkt(pkt, pktlen, XGS_DMA_RX_CHAN); 3536 3537 if (device_is_dpp(sinfo)) { 3538 uint16_t tpid = 0; 3539 uint16_t vid = 0; 3540 int res = -1; 3541 3542 memset(&packet_info, 0, sizeof(bkn_dune_system_header_info_t)); 3543 res = bkn_dpp_packet_header_parse(sinfo, pkt, (uint32_t)pktlen, &packet_info); 3544 if (res == 0) { 3545 if (packet_info.ftmh.action_type == 0x2) { 3546 bkn_bitstream_set_field((uint32_t *)dcb, 192, 32, 0x8); 3547 } else if (packet_info.ftmh.action_type == 0x1) { 3548 bkn_bitstream_set_field((uint32_t *)dcb, 231, 25, 0x20); 3549 } 3550 bkn_bitstream_set_field((uint32_t *)dcb, 112, 16, packet_info.ftmh.src_sys_port); 3551 bkn_bitstream_set_field((uint32_t *)dcb, 296, 12, packet_info.internal.vsi); 3552 bkn_bitstream_set_field((uint32_t *)dcb, 64, 32, (packet_info.internal.trap_id << 16 | packet_info.internal.trap_qualifier)); 3553 pkt += packet_info.ntwrk_header_ptr; 3554 pktlen -= packet_info.ntwrk_header_ptr; 3555 bkn_dump_pkt(pkt, 32, XGS_DMA_RX_CHAN); 3556 /* check if vlan tag exists */ 3557 tpid = (uint16_t)((pkt[12] << 8) | pkt[13]); 3558 vid = (uint16_t)(packet_info.internal.vsi & 0xfff); 3559 if (packet_is_untagged(tpid)) { 3560 if ((pktlen + 4) < rx_buffer_size) { 3561 DBG_DUNE(("add vlan tag (%d) to untagged packets\n", vid)); 3562 for (idx = (pktlen-1); idx >= 12; idx--) { 3563 pkt[idx+4] = pkt[idx]; 3564 } 3565 pkt[12] = 0x81; 3566 pkt[13] = 0x00; 3567 pkt[14] = (vid >> 8); 3568 pkt[15] = (vid & 0xff); 3569 /* reset packet length in DCB */ 3570 pktlen += 4; 3571 dcb[sinfo->dcb_wsize-1] &= ~SOC_DCB_KNET_COUNT_MASK; 3572 dcb[sinfo->dcb_wsize-1] |= ((pktlen + packet_info.ntwrk_header_ptr) & SOC_DCB_KNET_COUNT_MASK); 3573 bkn_dump_pkt(pkt, 32, XGS_DMA_RX_CHAN); 3574 } 3575 } 3576 } 3577 } 3578 else if (device_is_dnx(sinfo)) { 3579 int res = -1; 3580 res = bkn_dnx_packet_header_parse(sinfo, pkt, (uint32_t)pktlen, &packet_info); 3581 if (res == 0) { 3582 bkn_bitstream_set_field(meta, 0, 16, packet_info.fhei_code); 3583 bkn_bitstream_set_field(meta, 16, 16, packet_info.fhei_qualifier); 3584 bkn_bitstream_set_field(meta, 32, 16, packet_info.ftmh_spa); 3585 bkn_bitstream_set_field(meta, 48, 16, (packet_info.pph_forward_domain & 0xffff)); 3586 3587 pkt += packet_info.system_header_size; 3588 pktlen -= packet_info.system_header_size; 3589 bkn_dump_pkt(pkt, 32, XGS_DMA_RX_CHAN); 3590 } 3591 } 3592 3593 if (device_is_dpp(sinfo)) { 3594 filter = bkn_match_rx_pkt(sinfo, pkt, pktlen, dcb, chan, &cbf); 3595 } else if (device_is_dnx(sinfo)) { 3596 filter = bkn_match_rx_pkt(sinfo, pkt, pktlen, meta, chan, &cbf); 3597 } else { 3598 filter = bkn_match_rx_pkt(sinfo, pkt + sinfo->pkt_hdr_size, 3599 pktlen - sinfo->pkt_hdr_size, meta, chan, &cbf); 3600 } 3601 if ((dcb[sinfo->dcb_wsize-1] & 0xf0000) != 0x30000) { 3602 /* Fragment or error */ 3603 if (filter && filter->kf.mask.w[err_woff] == 0) { 3604 /* Drop unless DCB status is part of filter */ 3605 filter = NULL; 3606 } 3607 } 3608 drop_api = 1; 3609 if (filter) { 3610 DBG_FLTR(("Match filter ID %d\n", filter->kf.id)); 3611 switch (filter->kf.dest_type) { 3612 case KCOM_DEST_T_API: 3613 DBG_FLTR(("Send to Rx API\n")); 3614 sinfo->rx[chan].pkts_f_api++; 3615 drop_api = 0; 3616 break; 3617 case KCOM_DEST_T_NETIF: 3618 priv = bkn_netif_lookup(sinfo, filter->kf.dest_id); 3619 if (priv) { 3620 /* Check that software link is up */ 3621 if (!netif_carrier_ok(priv->dev)) { 3622 sinfo->rx[chan].pkts_d_no_link++; 3623 break; 3624 } 3625 3626 if (sinfo->cmic_type == 'x') { 3627 pkt += sinfo->pkt_hdr_size; 3628 pktlen -= sinfo->pkt_hdr_size; 3629 } 3630 3631 /* Add 2 bytes for IP header alignment (see below) */ 3632 skb = dev_alloc_skb(pktlen + RCPU_RX_ENCAP_SIZE + 2); 3633 if (skb == NULL) { 3634 sinfo->rx[chan].pkts_d_no_skb++; 3635 break; 3636 } 3637 skb_reserve(skb, RCPU_RX_ENCAP_SIZE); 3638 3639 DBG_FLTR(("Send to netif %d (%s)\n", 3640 priv->id, priv->dev->name)); 3641 sinfo->rx[chan].pkts_f_netif++; 3642 skb->dev = priv->dev; 3643 skb_reserve(skb, 2); /* 16 byte align the IP fields. */ 3644 3645 /* Save for RCPU before stripping tag */ 3646 ethertype = (pkt[16] << 8) | pkt[17]; 3647 if ((priv->flags & KCOM_NETIF_F_KEEP_RX_TAG) == 0) { 3648 if (filter->kf.flags & KCOM_FILTER_F_STRIP_TAG) { 3649 /* Strip the VLAN tag */ 3650 uint16_t vlan_proto = (uint16_t)((pkt[12] << 8) | pkt[13]); 3651 if (vlan_proto == 0x8100 || vlan_proto == 0x88a8) { 3652 DBG_FLTR(("Strip VLAN tag\n")); 3653 for (idx = 11; idx >= 0; idx--) { 3654 pkt[idx+4] = pkt[idx]; 3655 } 3656 pktlen -= 4; 3657 pkt += 4; 3658 } 3659 } else { 3660 /* 3661 * Mark packet as VLAN-tagged, otherwise newer 3662 * kernels will strip the tag. 3663 */ 3664 uint16_t tci = (pkt[14] << 8) | pkt[15]; 3665 if (priv->flags & KCOM_NETIF_F_RCPU_ENCAP) { 3666 bkn_vlan_hwaccel_put_tag(skb, ETH_P_8021Q, tci); 3667 } else { 3668 bkn_vlan_hwaccel_put_tag(skb, 3669 ((skb->data[12] << 8) | skb->data[13]), tci); 3670 } 3671 } 3672 } 3673 3674 skb_copy_to_linear_data(skb, pkt, pktlen); 3675 if (device_is_sand(sinfo)) { 3676 /* CRC has been stripped */ 3677 skb_put(skb, pktlen); 3678 } else { 3679 skb_put(skb, pktlen - 4); /* Strip CRC */ 3680 } 3681 priv->stats.rx_packets++; 3682 priv->stats.rx_bytes += skb->len; 3683 3684 /* Optional SKB updates */ 3685 if (knet_rx_cb != NULL) { 3686 KNET_SKB_CB(skb)->netif_user_data = priv->cb_user_data; 3687 KNET_SKB_CB(skb)->filter_user_data = filter->kf.cb_user_data; 3688 KNET_SKB_CB(skb)->dcb_type = sinfo->dcb_type & 0xFFFF; 3689 skb = knet_rx_cb(skb, sinfo->dev_no, meta); 3690 if (skb == NULL) { 3691 /* Consumed by call-back */ 3692 sinfo->rx[chan].pkts_d_callback++; 3693 break; 3694 } 3695 } 3696 3697 /* Do Rx timestamping */ 3698 if (sinfo->rx_hwts) { 3699 bkn_hw_tstamp_rx_set(sinfo, skb, meta); 3700 } 3701 3702 if (priv->flags & KCOM_NETIF_F_RCPU_ENCAP) { 3703 bkn_add_rcpu_encap(sinfo, skb, meta); 3704 DBG_PDMP(("After add RCPU ENCAP\n")); 3705 bkn_dump_pkt(skb->data, pktlen + RCPU_RX_ENCAP_SIZE, XGS_DMA_RX_CHAN); 3706 } 3707 skb->protocol = eth_type_trans(skb, skb->dev); 3708 if (filter->kf.dest_proto) { 3709 skb->protocol = filter->kf.dest_proto; 3710 } 3711 if (priv->flags & KCOM_NETIF_F_RCPU_ENCAP) { 3712 bkn_eth_type_update(skb, ethertype); 3713 } 3714 DBG_DUNE(("skb protocol 0x%04x\n",skb->protocol)); 3715 3716 /* Unlock while calling up network stack */ 3717 spin_unlock(&sinfo->lock); 3718 if (use_napi) { 3719 netif_receive_skb(skb); 3720 } else { 3721 netif_rx(skb); 3722 } 3723 spin_lock(&sinfo->lock); 3724 3725 if (filter->kf.mirror_type == KCOM_DEST_T_API || 3726 dbg_pkt_enable) { 3727 DBG_FLTR(("Mirror to Rx API\n")); 3728 sinfo->rx[chan].pkts_m_api++; 3729 drop_api = 0; 3730 } 3731 } else { 3732 DBG_FLTR(("Unknown netif %d\n", 3733 filter->kf.dest_id)); 3734 sinfo->rx[chan].pkts_d_unkn_netif++; 3735 } 3736 break; 3737 default: 3738 /* Drop packet */ 3739 DBG_FLTR(("Unknown dest type %d\n", 3740 filter->kf.dest_type)); 3741 sinfo->rx[chan].pkts_d_unkn_dest++; 3742 break; 3743 } 3744 } else { 3745 DBG_PKT(("Rx packet dropped.\n")); 3746 sinfo->rx[chan].pkts_d_no_match++; 3747 } 3748 if (drop_api) { 3749 /* If count is zero, the DCB will just be recycled */ 3750 dcb[sinfo->dcb_wsize-1] &= ~SOC_DCB_KNET_COUNT_MASK; 3751 } 3752 dcb[sinfo->dcb_wsize-1] |= SOC_DCB_KNET_DONE; 3753 dcb_chain->dcb_cur++; 3754 dcbs_done++; 3755 } 3756 3757 return dcbs_done; 3758 } 3759 3760 static int 3761 bkn_do_skb_rx(bkn_switch_info_t *sinfo, int chan, int budget) 3762 { 3763 bkn_priv_t *priv; 3764 bkn_desc_info_t *desc; 3765 struct sk_buff *skb; 3766 bkn_filter_t cbf; 3767 bkn_filter_t *filter; 3768 uint32_t err_woff; 3769 uint32_t *dcb, *meta; 3770 int ethertype; 3771 int pktlen; 3772 int idx; 3773 int dcbs_done = 0; 3774 bkn_dune_system_header_info_t packet_info = {0}; 3775 uint32_t dnx_meta_data[3] = {0}; 3776 3777 if (!sinfo->rx[chan].running) { 3778 /* Rx not ready */ 3779 return 0; 3780 } 3781 3782 while (dcbs_done < budget) { 3783 char str[32]; 3784 sprintf(str, "Rx DCB (%d)", sinfo->rx[chan].dirty); 3785 desc = &sinfo->rx[chan].desc[sinfo->rx[chan].dirty]; 3786 dcb = desc->dcb_mem; 3787 bkn_dump_dcb(str, dcb, sinfo->dcb_wsize, XGS_DMA_RX_CHAN); 3788 if ((dcb[sinfo->dcb_wsize-1] & (1 << 31)) == 0) { 3789 break; 3790 } 3791 if ((sinfo->cmic_type == 'x' && (dcb[2] & (1 << 16)) == 0) || 3792 (sinfo->cmic_type != 'x' && (dcb[1] & (1 << 16)) == 0)) { 3793 sinfo->rx[chan].chain_complete = 1; 3794 /* Request one extra poll to check for chain done interrupt */ 3795 if (sinfo->napi_poll_mode) { 3796 sinfo->napi_poll_again = 1; 3797 } 3798 } 3799 sinfo->rx[chan].pkts++; 3800 skb = desc->skb; 3801 if (sinfo->cmic_type == 'x') { 3802 if (device_is_dnx(sinfo)){ 3803 meta = &dnx_meta_data[0]; 3804 /* get error bit from last DCB words */ 3805 err_woff = 2; 3806 meta[err_woff] = dcb[sinfo->dcb_wsize-1]; 3807 } else { 3808 meta = (uint32_t *)skb->data; 3809 err_woff = sinfo->pkt_hdr_size / sizeof(uint32_t) - 1; 3810 meta[err_woff] = dcb[sinfo->dcb_wsize-1]; 3811 } 3812 } else { 3813 meta = dcb; 3814 err_woff = sinfo->dcb_wsize - 1; 3815 } 3816 pktlen = dcb[sinfo->dcb_wsize-1] & 0xffff; 3817 priv = netdev_priv(sinfo->dev); 3818 DBG_DCB_RX(("Rx%d SKB DMA done (%d).\n", chan, sinfo->rx[chan].dirty)); 3819 DMA_UNMAP_SINGLE(sinfo->dma_dev, 3820 desc->skb_dma, desc->dma_size, 3821 DMA_FROMDEV); 3822 desc->skb_dma = 0; 3823 bkn_dump_pkt(skb->data, pktlen, XGS_DMA_RX_CHAN); 3824 3825 if (device_is_dpp(sinfo)) { 3826 uint16_t tpid = 0; 3827 uint16_t vid = 0; 3828 uint8_t *pkt = skb->data; 3829 int res = 0; 3830 3831 memset(&packet_info, 0, sizeof(bkn_dune_system_header_info_t)); 3832 res = bkn_dpp_packet_header_parse(sinfo, pkt, (uint32_t)pktlen, &packet_info); 3833 if (res == 0) { 3834 if (packet_info.ftmh.action_type == 0x2) { 3835 bkn_bitstream_set_field((uint32_t *)dcb, 192, 32, 0x8); 3836 } else if (packet_info.ftmh.action_type == 0x1) { 3837 bkn_bitstream_set_field((uint32_t *)dcb, 231, 25, 0x20); 3838 } 3839 bkn_bitstream_set_field((uint32_t *)dcb, 112, 16, packet_info.ftmh.src_sys_port); 3840 bkn_bitstream_set_field((uint32_t *)dcb, 296, 12, packet_info.internal.vsi); 3841 bkn_bitstream_set_field((uint32_t *)dcb, 64, 32, (packet_info.internal.trap_id << 16 | packet_info.internal.trap_qualifier)); 3842 pkt = skb->data + packet_info.ntwrk_header_ptr; 3843 /* check if vlan tag exists */ 3844 tpid = (uint16_t)((pkt[12] << 8) | pkt[13]); 3845 vid = (uint16_t)(packet_info.internal.vsi & 0xfff); 3846 if (packet_is_untagged(tpid)) { 3847 if ((pktlen + 4) < rx_buffer_size) { 3848 DBG_DUNE(("add vlan tag to untagged packets\n")); 3849 for (idx = (pktlen-packet_info.ntwrk_header_ptr-1); idx >= 12; idx--) { 3850 pkt[idx+4] = pkt[idx]; 3851 } 3852 pkt[12] = 0x81; 3853 pkt[13] = 0x00; 3854 pkt[14] = (vid >> 8); 3855 pkt[15] = (vid & 0xff); 3856 pktlen += 4; 3857 /* reset packet length in DCB */ 3858 dcb[sinfo->dcb_wsize-1] &= ~SOC_DCB_KNET_COUNT_MASK; 3859 dcb[sinfo->dcb_wsize-1] |= (pktlen & SOC_DCB_KNET_COUNT_MASK); 3860 bkn_dump_pkt(pkt, 32, XGS_DMA_RX_CHAN); 3861 } 3862 } 3863 } 3864 } 3865 else if (device_is_dnx(sinfo)) { 3866 uint8_t *pkt = skb->data; 3867 int res = -1; 3868 3869 res = bkn_dnx_packet_header_parse(sinfo, pkt, (uint32_t)pktlen, &packet_info); 3870 if (res == 0) { 3871 bkn_bitstream_set_field(meta, 0, 16, packet_info.fhei_code); 3872 bkn_bitstream_set_field(meta, 16, 16, packet_info.fhei_qualifier); 3873 bkn_bitstream_set_field(meta, 32, 16, packet_info.ftmh_spa); 3874 bkn_bitstream_set_field(meta, 48, 16, (packet_info.pph_forward_domain & 0xffff)); 3875 3876 pkt += packet_info.system_header_size; 3877 pktlen -= packet_info.ntwrk_header_ptr; 3878 bkn_dump_pkt(pkt, 32, XGS_DMA_RX_CHAN); 3879 } 3880 } 3881 3882 if (device_is_dpp(sinfo)) { 3883 filter = bkn_match_rx_pkt(sinfo, skb->data + packet_info.ntwrk_header_ptr, 3884 pktlen, dcb, chan, &cbf); 3885 } else if (device_is_dnx(sinfo)) { 3886 filter = bkn_match_rx_pkt(sinfo, skb->data + packet_info.system_header_size, 3887 pktlen, meta, chan, &cbf); 3888 } else { 3889 filter = bkn_match_rx_pkt(sinfo, skb->data + sinfo->pkt_hdr_size, 3890 pktlen - sinfo->pkt_hdr_size, meta, chan, &cbf); 3891 } 3892 if ((dcb[sinfo->dcb_wsize-1] & 0xf0000) != 0x30000) { 3893 /* Fragment or error */ 3894 priv->stats.rx_errors++; 3895 if (filter && filter->kf.mask.w[err_woff] == 0) { 3896 /* Drop unless DCB status is part of filter */ 3897 filter = NULL; 3898 } 3899 } 3900 DBG_PKT(("Rx packet (%d bytes).\n", pktlen)); 3901 if (filter) { 3902 DBG_FLTR(("Match filter ID %d\n", filter->kf.id)); 3903 switch (filter->kf.dest_type) { 3904 case KCOM_DEST_T_API: 3905 DBG_FLTR(("Send to Rx API\n")); 3906 sinfo->rx[chan].pkts_f_api++; 3907 bkn_api_rx_copy_from_skb(sinfo, chan, desc); 3908 break; 3909 case KCOM_DEST_T_NETIF: 3910 priv = bkn_netif_lookup(sinfo, filter->kf.dest_id); 3911 if (priv) { 3912 /* Check that software link is up */ 3913 if (!netif_carrier_ok(priv->dev)) { 3914 sinfo->rx[chan].pkts_d_no_link++; 3915 break; 3916 } 3917 DBG_FLTR(("Send to netif %d (%s)\n", 3918 priv->id, priv->dev->name)); 3919 sinfo->rx[chan].pkts_f_netif++; 3920 3921 if (((filter->kf.mirror_type == KCOM_DEST_T_API) && 3922 (!device_is_sand(sinfo))) || dbg_pkt_enable) { 3923 sinfo->rx[chan].pkts_m_api++; 3924 bkn_api_rx_copy_from_skb(sinfo, chan, desc); 3925 } 3926 3927 if (device_is_dpp(sinfo)) { 3928 if (filter->kf.mirror_type == KCOM_DEST_T_API) { 3929 sinfo->rx[chan].pkts_m_api++; 3930 bkn_api_rx_copy_from_skb(sinfo, chan, desc); 3931 } 3932 /* Strip Dune headers */ 3933 skb->data += packet_info.ntwrk_header_ptr; 3934 pktlen -= packet_info.ntwrk_header_ptr; 3935 bkn_dump_pkt(skb->data, 32, XGS_DMA_RX_CHAN); 3936 /* CRC has been stripped on Dune*/ 3937 skb_put(skb, pktlen); 3938 } else if (device_is_dnx(sinfo)) { 3939 if (filter->kf.mirror_type == KCOM_DEST_T_API) { 3940 sinfo->rx[chan].pkts_m_api++; 3941 bkn_api_rx_copy_from_skb(sinfo, chan, desc); 3942 } 3943 /* Strip Dune headers */ 3944 skb->data += packet_info.system_header_size; 3945 pktlen -= packet_info.system_header_size; 3946 bkn_dump_pkt(skb->data, 32, XGS_DMA_RX_CHAN); 3947 /* CRC has been stripped on Dune*/ 3948 skb_put(skb, pktlen); 3949 } else { 3950 skb_put(skb, pktlen - 4); /* Strip CRC */ 3951 } 3952 3953 if (sinfo->cmic_type == 'x' && !device_is_dnx(sinfo)) { 3954 skb_pull(skb, sinfo->pkt_hdr_size); 3955 } 3956 3957 /* Save for RCPU before stripping tag */ 3958 ethertype = (skb->data[16] << 8) | skb->data[17]; 3959 if ((priv->flags & KCOM_NETIF_F_KEEP_RX_TAG) == 0) { 3960 if (filter->kf.flags & KCOM_FILTER_F_STRIP_TAG) { 3961 /* Strip VLAN tag */ 3962 uint16_t vlan_proto = (uint16_t)((skb->data[12] << 8) | skb->data[13]); 3963 if (vlan_proto == 0x8100 || vlan_proto == 0x88a8) { 3964 DBG_FLTR(("Strip VLAN tag\n")); 3965 ((u32*)skb->data)[3] = ((u32*)skb->data)[2]; 3966 ((u32*)skb->data)[2] = ((u32*)skb->data)[1]; 3967 ((u32*)skb->data)[1] = ((u32*)skb->data)[0]; 3968 skb_pull(skb, 4); 3969 if (sinfo->cmic_type == 'x' && !device_is_dnx(sinfo)) { 3970 for (idx = sinfo->pkt_hdr_size / sizeof(uint32_t); idx; idx--) { 3971 meta[idx] = meta[idx - 1]; 3972 } 3973 meta++; 3974 } 3975 } 3976 } else { 3977 /* 3978 * Mark packet as VLAN-tagged, otherwise newer 3979 * kernels will strip the tag. 3980 */ 3981 uint16_t tci = (skb->data[14] << 8) | skb->data[15]; 3982 if (priv->flags & KCOM_NETIF_F_RCPU_ENCAP) { 3983 bkn_vlan_hwaccel_put_tag(skb, ETH_P_8021Q, tci); 3984 } else { 3985 bkn_vlan_hwaccel_put_tag(skb, 3986 ((skb->data[12] << 8) | skb->data[13]), tci); 3987 } 3988 } 3989 } 3990 priv->stats.rx_packets++; 3991 priv->stats.rx_bytes += skb->len; 3992 skb->dev = priv->dev; 3993 3994 /* Optional SKB updates */ 3995 if (knet_rx_cb != NULL) { 3996 KNET_SKB_CB(skb)->netif_user_data = priv->cb_user_data; 3997 KNET_SKB_CB(skb)->filter_user_data = filter->kf.cb_user_data; 3998 KNET_SKB_CB(skb)->dcb_type = sinfo->dcb_type & 0xFFFF; 3999 skb = knet_rx_cb(skb, sinfo->dev_no, meta); 4000 if (skb == NULL) { 4001 /* Consumed by call-back */ 4002 sinfo->rx[chan].pkts_d_callback++; 4003 priv->stats.rx_dropped++; 4004 desc->skb = NULL; 4005 break; 4006 } 4007 } 4008 4009 /* Do Rx timestamping */ 4010 if (sinfo->rx_hwts) { 4011 bkn_hw_tstamp_rx_set(sinfo, skb, meta); 4012 } 4013 4014 if (priv->flags & KCOM_NETIF_F_RCPU_ENCAP) { 4015 bkn_add_rcpu_encap(sinfo, skb, meta); 4016 DBG_PDMP(("After add RCPU ENCAP\n")); 4017 bkn_dump_pkt(skb->data, pktlen + RCPU_RX_ENCAP_SIZE, XGS_DMA_RX_CHAN); 4018 } 4019 skb->protocol = eth_type_trans(skb, skb->dev); 4020 if (filter->kf.dest_proto) { 4021 skb->protocol = filter->kf.dest_proto; 4022 } 4023 if (priv->flags & KCOM_NETIF_F_RCPU_ENCAP) { 4024 bkn_eth_type_update(skb, ethertype); 4025 } 4026 DBG_DUNE(("skb protocol 0x%04x\n",skb->protocol)); 4027 4028 if (filter->kf.mirror_type == KCOM_DEST_T_NETIF) { 4029 bkn_priv_t *mpriv; 4030 struct sk_buff *mskb; 4031 mpriv = bkn_netif_lookup(sinfo, filter->kf.mirror_id); 4032 if (mpriv && netif_carrier_ok(mpriv->dev)) { 4033 mskb = skb_clone(skb, GFP_ATOMIC); 4034 if (mskb == NULL) { 4035 sinfo->rx[chan].pkts_d_no_skb++; 4036 } else { 4037 sinfo->rx[chan].pkts_m_netif++; 4038 mpriv->stats.rx_packets++; 4039 mpriv->stats.rx_bytes += mskb->len; 4040 skb->dev = mpriv->dev; 4041 if (filter->kf.mirror_proto) { 4042 skb->protocol = filter->kf.mirror_proto; 4043 } 4044 /* Unlock while calling up network stack */ 4045 spin_unlock(&sinfo->lock); 4046 if (use_napi) { 4047 netif_receive_skb(mskb); 4048 } else { 4049 netif_rx(mskb); 4050 } 4051 spin_lock(&sinfo->lock); 4052 } 4053 } 4054 } 4055 4056 /* Unlock while calling up network stack */ 4057 spin_unlock(&sinfo->lock); 4058 if (use_napi) { 4059 netif_receive_skb(skb); 4060 } else { 4061 netif_rx(skb); 4062 } 4063 spin_lock(&sinfo->lock); 4064 4065 /* Ensure that we reallocate SKB for this DCB */ 4066 desc->skb = NULL; 4067 } else { 4068 DBG_FLTR(("Unknown netif %d\n", 4069 filter->kf.dest_id)); 4070 sinfo->rx[chan].pkts_d_unkn_netif++; 4071 } 4072 break; 4073 default: 4074 /* Drop packet */ 4075 DBG_FLTR(("Unknown dest type %d\n", 4076 filter->kf.dest_type)); 4077 sinfo->rx[chan].pkts_d_unkn_dest++; 4078 break; 4079 } 4080 } else { 4081 DBG_PKT(("Rx packet dropped.\n")); 4082 sinfo->rx[chan].pkts_d_no_match++; 4083 priv->stats.rx_dropped++; 4084 } 4085 dcb[sinfo->dcb_wsize-1] &= ~(1 << 31); 4086 if (++sinfo->rx[chan].dirty >= MAX_RX_DCBS) { 4087 sinfo->rx[chan].dirty = 0; 4088 } 4089 sinfo->rx[chan].free--; 4090 dcbs_done++; 4091 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 4092 /* Right now refill for Continuous DMA mode */ 4093 bkn_rx_refill(sinfo, chan); 4094 } 4095 } 4096 4097 return dcbs_done; 4098 } 4099 4100 static int 4101 bkn_do_rx(bkn_switch_info_t *sinfo, int chan, int budget) 4102 { 4103 if (sinfo->rx[chan].use_rx_skb == 0) { 4104 /* Rx buffers are provided by BCM Rx API */ 4105 return bkn_do_api_rx(sinfo, chan, budget); 4106 } else { 4107 /* Rx buffers are provided by Linux kernel */ 4108 return bkn_do_skb_rx(sinfo, chan, budget); 4109 } 4110 } 4111 4112 static void 4113 bkn_rx_desc_done(bkn_switch_info_t *sinfo, int chan) 4114 { 4115 bkn_evt_resource_t *evt; 4116 evt = &_bkn_evt[sinfo->evt_idx]; 4117 DBG_IRQ(("Rx%d desc done\n", chan)); 4118 4119 if (sinfo->rx[chan].use_rx_skb == 0) { 4120 sinfo->dma_events |= KCOM_DMA_INFO_F_RX_DONE; 4121 evt->evt_wq_put++; 4122 wake_up_interruptible(&evt->evt_wq); 4123 } 4124 } 4125 4126 static void 4127 bkn_rx_chain_done(bkn_switch_info_t *sinfo, int chan) 4128 { 4129 DBG_IRQ(("Rx%d chain done\n", chan)); 4130 4131 if (sinfo->rx[chan].chain_complete == 0) { 4132 /* 4133 * In certain environments the DCB memory is updated after 4134 * the corresponding interrupt has been received. 4135 * The following code will ensure that this situation is 4136 * handled properly. 4137 */ 4138 int maxloop = 0; 4139 while (sinfo->rx[chan].chain_complete == 0) { 4140 sinfo->rx[chan].sync_retry++; 4141 if (maxloop == 0) { 4142 sinfo->rx[chan].sync_err++; 4143 } 4144 if (maxloop > sinfo->rx[chan].sync_maxloop) { 4145 sinfo->rx[chan].sync_maxloop = maxloop; 4146 } 4147 if (bkn_do_rx(sinfo, chan, MAX_RX_DCBS) > 0) { 4148 bkn_rx_desc_done(sinfo, chan); 4149 } 4150 if (++maxloop > rx_sync_retry) { 4151 gprintk("Fatal error: Incomplete chain\n"); 4152 sinfo->rx[chan].chain_complete = 1; 4153 break; 4154 } 4155 } 4156 } 4157 4158 sinfo->rx[chan].running = 0; 4159 4160 if (sinfo->rx[chan].use_rx_skb == 0) { 4161 bkn_api_rx_chain_done(sinfo, chan); 4162 } else { 4163 bkn_rx_refill(sinfo, chan); 4164 4165 if (bkn_rx_restart(sinfo, chan) != 0) { 4166 /* Presumably out of resources */ 4167 sinfo->timer.expires = jiffies + 1; 4168 if (!sinfo->timer_queued) { 4169 sinfo->timer_queued = 1; 4170 add_timer(&sinfo->timer); 4171 } 4172 } 4173 } 4174 } 4175 4176 static void 4177 bkn_suspend_tx(bkn_switch_info_t *sinfo) 4178 { 4179 struct list_head *list; 4180 bkn_priv_t *priv = netdev_priv(sinfo->dev); 4181 4182 /* Stop main device */ 4183 netif_stop_queue(priv->dev); 4184 sinfo->tx.suspends++; 4185 /* Stop associated virtual devices */ 4186 list_for_each(list, &sinfo->ndev_list) { 4187 priv = (bkn_priv_t *)list; 4188 netif_stop_queue(priv->dev); 4189 } 4190 } 4191 4192 static void 4193 bkn_resume_tx(bkn_switch_info_t *sinfo) 4194 { 4195 struct list_head *list; 4196 bkn_priv_t *priv = netdev_priv(sinfo->dev); 4197 4198 /* Check main device */ 4199 if (netif_queue_stopped(priv->dev) && sinfo->tx.free > 1) { 4200 netif_wake_queue(priv->dev); 4201 } 4202 /* Check associated virtual devices */ 4203 list_for_each(list, &sinfo->ndev_list) { 4204 priv = (bkn_priv_t *)list; 4205 if (netif_queue_stopped(priv->dev) && sinfo->tx.free > 1) { 4206 netif_wake_queue(priv->dev); 4207 } 4208 } 4209 } 4210 4211 static int 4212 bkn_hw_tstamp_tx_set(bkn_switch_info_t *sinfo, struct sk_buff *skb) 4213 { 4214 struct skb_shared_hwtstamps shhwtstamps; 4215 uint64_t ts = 0; 4216 uint32_t hdrlen = sinfo->cmic_type == 'x' ? PKT_TX_HDR_SIZE : 0; 4217 int port; 4218 4219 if (!knet_hw_tstamp_tx_time_get_cb) { 4220 return -1; 4221 } 4222 4223 port = KNET_SKB_CB(skb)->port; 4224 if (knet_hw_tstamp_tx_time_get_cb(sinfo->dev_no, port, skb->data + hdrlen, &ts) < 0) { 4225 return -1; 4226 } 4227 4228 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 4229 shhwtstamps.hwtstamp = ns_to_ktime(be64_to_cpu(ts)); 4230 skb_tstamp_tx(skb, &shhwtstamps); 4231 4232 return 0; 4233 } 4234 4235 static void 4236 bkn_hw_tstamp_tx_work(struct work_struct *work) 4237 { 4238 bkn_switch_info_t *sinfo = container_of(work, bkn_switch_info_t, tx_ptp_work); 4239 struct sk_buff *skb; 4240 4241 while (skb_queue_len(&sinfo->tx_ptp_queue)) { 4242 skb = skb_dequeue(&sinfo->tx_ptp_queue); 4243 if (bkn_hw_tstamp_tx_set(sinfo, skb) < 0) { 4244 gprintk("Timestamp has not been taken for the current skb.\n"); 4245 } 4246 dev_kfree_skb_any(skb); 4247 } 4248 } 4249 4250 static int 4251 bkn_do_tx(bkn_switch_info_t *sinfo) 4252 { 4253 bkn_desc_info_t *desc; 4254 int dcbs_done = 0; 4255 4256 if (!CDMA_CH(sinfo, XGS_DMA_TX_CHAN) && sinfo->tx.api_active) { 4257 return dcbs_done; 4258 } 4259 4260 while (dcbs_done < MAX_TX_DCBS) { 4261 char str[32]; 4262 if (sinfo->tx.free == MAX_TX_DCBS) { 4263 break; 4264 } 4265 sprintf(str, "Tx DCB (%d)", sinfo->tx.dirty); 4266 desc = &sinfo->tx.desc[sinfo->tx.dirty]; 4267 bkn_dump_dcb(str, desc->dcb_mem, sinfo->dcb_wsize, XGS_DMA_TX_CHAN); 4268 if ((desc->dcb_mem[sinfo->dcb_wsize-1] & (1 << 31)) == 0) { 4269 break; 4270 } 4271 if (desc->skb) { 4272 DBG_DCB_TX(("Tx SKB DMA done (%d).\n", sinfo->tx.dirty)); 4273 DMA_UNMAP_SINGLE(sinfo->dma_dev, 4274 desc->skb_dma, desc->dma_size, 4275 DMA_TODEV); 4276 if (bkn_skb_tx_flags(desc->skb) & SKBTX_IN_PROGRESS) { 4277 skb_queue_tail(&sinfo->tx_ptp_queue, desc->skb); 4278 schedule_work(&sinfo->tx_ptp_work); 4279 } else { 4280 dev_kfree_skb_any(desc->skb); 4281 } 4282 desc->skb = NULL; 4283 desc->skb_dma = 0; 4284 } 4285 desc->dcb_mem[sinfo->dcb_wsize-1] &= ~(1 << 31); 4286 if (++sinfo->tx.dirty >= MAX_TX_DCBS) { 4287 sinfo->tx.dirty = 0; 4288 } 4289 if (++sinfo->tx.free > MAX_TX_DCBS) { 4290 gprintk("Too many free Tx DCBs(%d).\n", sinfo->tx.free); 4291 } 4292 dcbs_done++; 4293 } 4294 4295 return dcbs_done; 4296 } 4297 4298 static void 4299 bkn_tx_cdma_chain_switch(bkn_switch_info_t *sinfo) 4300 { 4301 bkn_dcb_chain_t *dcb_chain = sinfo->tx.api_dcb_chain; 4302 uint32_t *dcb_mem; 4303 uint64_t dcb_dma; 4304 int woffset; 4305 4306 /* Switch between SKB Tx and API Tx for Continuous DMA mode */ 4307 if (!sinfo->tx.api_active) { 4308 /* 4309 * Set the current SKB DCB as reload DCB and the last DCB of 4310 * the pending API chain as the new halt location. 4311 */ 4312 sinfo->tx.api_active = 1; 4313 dcb_mem = sinfo->tx.desc[sinfo->tx.cur].dcb_mem; 4314 memset(dcb_mem, 0, sinfo->dcb_wsize * sizeof(uint32_t)); 4315 dcb_mem[0] = dcb_chain->dcb_dma; 4316 if (sinfo->cmic_type == 'x') { 4317 dcb_mem[1] = DMA_TO_BUS_HI(dcb_chain->dcb_dma >> 32); 4318 dcb_mem[2] |= 1 << 24 | 1 << 18 | 1 << 16; 4319 } else { 4320 dcb_mem[1] |= 1 << 24 | 1 << 18 | 1 << 16; 4321 } 4322 if (++sinfo->tx.cur >= MAX_TX_DCBS) { 4323 sinfo->tx.cur = 0; 4324 } 4325 sinfo->tx.free--; 4326 woffset = (dcb_chain->dcb_cnt - 1) * sinfo->dcb_wsize; 4327 dcb_dma = dcb_chain->dcb_dma + woffset * sizeof(uint32_t); 4328 /* DMA run to the new halt location */ 4329 bkn_cdma_goto(sinfo, XGS_DMA_TX_CHAN, dcb_dma); 4330 } else { 4331 /* Only need to set the current SKB DCB as the new halt location */ 4332 sinfo->tx.api_active = 0; 4333 woffset = (dcb_chain->dcb_cnt - 1) * sinfo->dcb_wsize; 4334 dcb_mem = &dcb_chain->dcb_mem[woffset]; 4335 dcb_mem[0] = sinfo->tx.desc[sinfo->tx.dirty].dcb_dma; 4336 if (sinfo->cmic_type == 'x') { 4337 dcb_mem[1] = DMA_TO_BUS_HI(sinfo->tx.desc[sinfo->tx.dirty].dcb_dma >> 32); 4338 } 4339 dcb_dma = sinfo->tx.desc[sinfo->tx.cur].dcb_dma; 4340 /* DMA run to the new halt location */ 4341 bkn_cdma_goto(sinfo, XGS_DMA_TX_CHAN, dcb_dma); 4342 } 4343 } 4344 4345 static void 4346 bkn_skb_tx(bkn_switch_info_t *sinfo) 4347 { 4348 if (sinfo->tx.api_active) { 4349 /* Switch from API Tx to SKB Tx */ 4350 bkn_tx_cdma_chain_switch(sinfo); 4351 } 4352 } 4353 4354 static void 4355 bkn_api_tx(bkn_switch_info_t *sinfo) 4356 { 4357 bkn_dcb_chain_t *dcb_chain; 4358 uint64_t pkt_dma; 4359 unsigned char *pktdata; 4360 int pktlen; 4361 int i; 4362 4363 /* Assume that driver lock is held */ 4364 if (list_empty(&sinfo->tx.api_dcb_list)) { 4365 sinfo->tx.api_active = 0; 4366 } else { 4367 sinfo->tx.pkts++; 4368 dcb_chain = list_entry(sinfo->tx.api_dcb_list.next, 4369 bkn_dcb_chain_t, list); 4370 DBG_DCB_TX(("Start API Tx DMA, first DCB @ 0x%08x (%d DCBs).\n", 4371 (uint32_t)dcb_chain->dcb_dma, dcb_chain->dcb_cnt)); 4372 for (i = 0; i < dcb_chain->dcb_cnt && debug & DBG_LVL_PDMP; i++) { 4373 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN) && i == dcb_chain->dcb_cnt - 1) { 4374 break; 4375 } 4376 if (sinfo->cmic_type == 'x') { 4377 pkt_dma = BUS_TO_DMA_HI(dcb_chain->dcb_mem[sinfo->dcb_wsize * i + 1]); 4378 pkt_dma = pkt_dma << 32 | dcb_chain->dcb_mem[sinfo->dcb_wsize * i]; 4379 } else { 4380 pkt_dma = dcb_chain->dcb_mem[sinfo->dcb_wsize * i]; 4381 } 4382 pktdata = kernel_bde->p2l(sinfo->dev_no, pkt_dma); 4383 if (sinfo->cmic_type == 'x') { 4384 pktlen = dcb_chain->dcb_mem[sinfo->dcb_wsize * i + 2] & 0xffff; 4385 } else { 4386 pktlen = dcb_chain->dcb_mem[sinfo->dcb_wsize * i + 1] & 0xffff; 4387 } 4388 bkn_dump_pkt(pktdata, pktlen, XGS_DMA_TX_CHAN); 4389 } 4390 4391 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN)) { 4392 sinfo->tx.api_dcb_chain = dcb_chain; 4393 if (!sinfo->tx.api_active) { 4394 /* Switch from SKB Tx to API Tx */ 4395 bkn_tx_cdma_chain_switch(sinfo); 4396 } 4397 list_del(&dcb_chain->list); 4398 } else { 4399 sinfo->tx.api_active = 1; 4400 dev_dma_chan_clear(sinfo, XGS_DMA_TX_CHAN); 4401 dev_irq_mask_enable(sinfo, XGS_DMA_TX_CHAN, 1); 4402 dev_dma_chan_start(sinfo, XGS_DMA_TX_CHAN, dcb_chain->dcb_dma); 4403 list_del(&dcb_chain->list); 4404 kfree(dcb_chain); 4405 } 4406 } 4407 } 4408 4409 static void 4410 bkn_tx_cdma_chain_done(bkn_switch_info_t *sinfo, int done) 4411 { 4412 int woffset; 4413 int dcbs_done = 0; 4414 bkn_evt_resource_t *evt; 4415 4416 DBG_IRQ(("Tx CDMA chain done \n")); 4417 4418 evt = &_bkn_evt[sinfo->evt_idx]; 4419 4420 if (sinfo->tx.api_active) { 4421 /* Drain API Tx chains */ 4422 while (sinfo->tx.api_dcb_chain != sinfo->tx.api_dcb_chain_end) { 4423 woffset = sinfo->tx.api_dcb_chain->dcb_cnt * sinfo->dcb_wsize - 1; 4424 if (!(sinfo->tx.api_dcb_chain->dcb_mem[woffset] & (1 << 31))) { 4425 return; 4426 } 4427 sinfo->tx.api_dcb_chain->dcb_mem[woffset - sinfo->dcb_wsize] |= SOC_DCB_KNET_DONE; 4428 sinfo->dma_events |= KCOM_DMA_INFO_F_TX_DONE; 4429 evt->evt_wq_put++; 4430 wake_up_interruptible(&evt->evt_wq); 4431 kfree(sinfo->tx.api_dcb_chain); 4432 sinfo->tx.api_dcb_chain = NULL; 4433 bkn_api_tx(sinfo); 4434 if ((++dcbs_done + done) >= MAX_TX_DCBS) { 4435 if (sinfo->napi_poll_mode) { 4436 /* Request one extra poll to reschedule Tx */ 4437 sinfo->napi_poll_again = 1; 4438 } else { 4439 /* Request to yield for Continuous DMA mode */ 4440 sinfo->tx_yield = 1; 4441 } 4442 return; 4443 } 4444 } 4445 woffset = (sinfo->tx.api_dcb_chain->dcb_cnt - 1) * sinfo->dcb_wsize - 1; 4446 if (!(sinfo->tx.api_dcb_chain->dcb_mem[woffset] & (1 << 31))) { 4447 return; 4448 } 4449 sinfo->tx.api_dcb_chain->dcb_mem[woffset] |= SOC_DCB_KNET_DONE; 4450 /* Try and park at SKB Tx if API Tx done */ 4451 bkn_skb_tx(sinfo); 4452 sinfo->dma_events |= KCOM_DMA_INFO_F_TX_DONE; 4453 evt->evt_wq_put++; 4454 wake_up_interruptible(&evt->evt_wq); 4455 kfree(sinfo->tx.api_dcb_chain); 4456 sinfo->tx.api_dcb_chain = NULL; 4457 sinfo->tx.api_dcb_chain_end = NULL; 4458 if (!sinfo->napi_poll_mode) { 4459 /* Not need to yield for Continuous DMA mode */ 4460 sinfo->tx_yield = 0; 4461 } 4462 } else { 4463 if (sinfo->tx.free == MAX_TX_DCBS) { 4464 /* Try API Tx if SKB Tx done */ 4465 bkn_api_tx(sinfo); 4466 if (sinfo->tx.api_active) { 4467 return; 4468 } 4469 } 4470 } 4471 4472 /* Resume if netif Tx resources available and API Tx not active */ 4473 bkn_resume_tx(sinfo); 4474 } 4475 4476 static void 4477 bkn_tx_chain_done(bkn_switch_info_t *sinfo, int done) 4478 { 4479 bkn_desc_info_t *desc; 4480 int idx, pending; 4481 bkn_evt_resource_t *evt; 4482 4483 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN)) { 4484 return bkn_tx_cdma_chain_done(sinfo, done); 4485 } 4486 4487 DBG_IRQ(("Tx chain done (%d/%d)\n", sinfo->tx.cur, sinfo->tx.dirty)); 4488 4489 dev_irq_mask_disable(sinfo, XGS_DMA_TX_CHAN, 0); 4490 4491 evt = &_bkn_evt[sinfo->evt_idx]; 4492 4493 if (sinfo->tx.api_active) { 4494 sinfo->dma_events |= KCOM_DMA_INFO_F_TX_DONE; 4495 evt->evt_wq_put++; 4496 wake_up_interruptible(&evt->evt_wq); 4497 /* Check if BCM API has more to send */ 4498 bkn_api_tx(sinfo); 4499 if (sinfo->tx.api_active) { 4500 return; 4501 } 4502 } 4503 4504 if (sinfo->tx.free == MAX_TX_DCBS) { 4505 /* If netif Tx is idle then allow BCM API to send */ 4506 bkn_api_tx(sinfo); 4507 if (sinfo->tx.api_active) { 4508 return; 4509 } 4510 } else { 4511 /* If two or more DCBs are pending, chain them */ 4512 pending = MAX_TX_DCBS - sinfo->tx.free; 4513 idx = sinfo->tx.dirty; 4514 while (--pending && idx < (MAX_TX_DCBS - 1)) { 4515 if (sinfo->cmic_type == 'x') { 4516 sinfo->tx.desc[idx++].dcb_mem[2] |= 1 << 16; 4517 } else { 4518 sinfo->tx.desc[idx++].dcb_mem[1] |= 1 << 16; 4519 } 4520 DBG_DCB_TX(("Chain Tx DCB %d (%d)\n", idx, pending)); 4521 } 4522 /* Restart DMA from where we stopped */ 4523 desc = &sinfo->tx.desc[sinfo->tx.dirty]; 4524 DBG_DCB_TX(("Restart Tx DMA, DCB @ 0x%08x (%d).\n", 4525 (uint32_t)desc->dcb_dma, sinfo->tx.dirty)); 4526 dev_dma_chan_clear(sinfo, XGS_DMA_TX_CHAN); 4527 dev_irq_mask_enable(sinfo, XGS_DMA_TX_CHAN, 0); 4528 dev_dma_chan_start(sinfo, XGS_DMA_TX_CHAN, desc->dcb_dma); 4529 } 4530 4531 /* Resume if netif Tx resources available and API Tx not active */ 4532 bkn_resume_tx(sinfo); 4533 } 4534 4535 static void 4536 bkn_schedule_napi_poll(bkn_switch_info_t *sinfo) 4537 { 4538 /* Schedule NAPI poll */ 4539 DBG_NAPI(("Schedule NAPI poll on %s.\n", sinfo->dev->name)); 4540 /* Disable interrupts until poll job is complete */ 4541 sinfo->napi_poll_mode = 1; 4542 /* Unlock while calling up network stack */ 4543 spin_unlock(&sinfo->lock); 4544 if (bkn_napi_schedule_prep(sinfo->dev, &sinfo->napi)) { 4545 __bkn_napi_schedule(sinfo->dev, &sinfo->napi); 4546 DBG_NAPI(("Schedule prep OK on %s.\n", sinfo->dev->name)); 4547 } else { 4548 /* Most likely the base device is has not been opened */ 4549 gprintk("Warning: Unable to schedule NAPI - base device not up?\n"); 4550 } 4551 spin_lock(&sinfo->lock); 4552 } 4553 4554 static void 4555 bkn_napi_poll_complete(bkn_switch_info_t *sinfo) 4556 { 4557 /* Unlock while calling up network stack */ 4558 spin_unlock(&sinfo->lock); 4559 bkn_napi_complete(sinfo->dev, &sinfo->napi); 4560 spin_lock(&sinfo->lock); 4561 /* Re-enable interrupts */ 4562 sinfo->napi_poll_mode = 0; 4563 dev_irq_mask_set(sinfo, sinfo->irq_mask); 4564 } 4565 4566 static int 4567 xgs_do_dma(bkn_switch_info_t *sinfo, int budget) 4568 { 4569 int rx_dcbs_done = 0, tx_dcbs_done = 0; 4570 int chan_done, budget_chans = 0; 4571 uint32_t dma_stat; 4572 int chan; 4573 4574 DEV_READ32(sinfo, CMIC_DMA_STATr, &dma_stat); 4575 4576 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4577 if (dma_stat & DS_DESC_DONE_TST(XGS_DMA_RX_CHAN + chan)) { 4578 xgs_dma_desc_clear(sinfo, XGS_DMA_RX_CHAN + chan); 4579 sinfo->poll_channels |= 1 << chan; 4580 } 4581 } 4582 if (!sinfo->poll_channels) { 4583 sinfo->poll_channels = (uint32_t)(1 << sinfo->rx_chans) - 1; 4584 budget_chans = budget / sinfo->rx_chans; 4585 } else { 4586 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4587 if (1 << chan & sinfo->poll_channels) { 4588 budget_chans++; 4589 } 4590 } 4591 budget_chans = budget / budget_chans; 4592 } 4593 4594 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4595 if (1 << chan & sinfo->poll_channels) { 4596 chan_done = bkn_do_rx(sinfo, chan, budget_chans); 4597 rx_dcbs_done += chan_done; 4598 if (chan_done < budget_chans) { 4599 sinfo->poll_channels &= ~(1 << chan); 4600 } 4601 bkn_rx_desc_done(sinfo, chan); 4602 } 4603 4604 if (dma_stat & DS_CHAIN_DONE_TST(XGS_DMA_RX_CHAN + chan)) { 4605 xgs_dma_chain_clear(sinfo, XGS_DMA_RX_CHAN + chan); 4606 bkn_rx_chain_done(sinfo, chan); 4607 } 4608 } 4609 4610 if (dma_stat & DS_CHAIN_DONE_TST(XGS_DMA_TX_CHAN)) { 4611 xgs_dma_chain_clear(sinfo, XGS_DMA_TX_CHAN); 4612 tx_dcbs_done = bkn_do_tx(sinfo); 4613 bkn_tx_chain_done(sinfo, tx_dcbs_done); 4614 } 4615 4616 return sinfo->poll_channels ? budget : rx_dcbs_done; 4617 } 4618 4619 static int 4620 xgsm_do_dma(bkn_switch_info_t *sinfo, int budget) 4621 { 4622 int rx_dcbs_done = 0, tx_dcbs_done = 0; 4623 int chan_done, budget_chans = 0; 4624 uint32_t dma_stat, irq_stat = 0; 4625 int chan; 4626 4627 /* Get Controlled interrupt states for Continuous DMA mode */ 4628 if (sinfo->cdma_channels) { 4629 DEV_READ32(sinfo, CMICM_IRQ_STATr, &irq_stat); 4630 } 4631 4632 DEV_READ32(sinfo, CMICM_DMA_STATr, &dma_stat); 4633 4634 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4635 if (dma_stat & (0x10 << (XGS_DMA_RX_CHAN + chan)) || 4636 irq_stat & (0x08000000 << (XGS_DMA_RX_CHAN + chan))) { 4637 xgsm_dma_desc_clear(sinfo, XGS_DMA_RX_CHAN + chan); 4638 sinfo->poll_channels |= 1 << chan; 4639 } 4640 } 4641 if (!sinfo->poll_channels) { 4642 sinfo->poll_channels = (uint32_t)(1 << sinfo->rx_chans) - 1; 4643 budget_chans = budget / sinfo->rx_chans; 4644 } else { 4645 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4646 if (1 << chan & sinfo->poll_channels) { 4647 budget_chans++; 4648 } 4649 } 4650 budget_chans = budget / budget_chans; 4651 } 4652 4653 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4654 if (1 << chan & sinfo->poll_channels) { 4655 chan_done = bkn_do_rx(sinfo, chan, budget_chans); 4656 rx_dcbs_done += chan_done; 4657 if (chan_done < budget_chans) { 4658 sinfo->poll_channels &= ~(1 << chan); 4659 } 4660 bkn_rx_desc_done(sinfo, chan); 4661 } 4662 4663 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 4664 continue; 4665 } 4666 4667 if (dma_stat & (0x1 << (XGS_DMA_RX_CHAN + chan))) { 4668 xgsm_dma_chain_clear(sinfo, XGS_DMA_RX_CHAN + chan); 4669 bkn_rx_chain_done(sinfo, chan); 4670 } 4671 } 4672 4673 if (dma_stat & (0x1 << XGS_DMA_TX_CHAN) || 4674 irq_stat & (0x08000000 << XGS_DMA_TX_CHAN)) { 4675 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN)) { 4676 xgsm_dma_desc_clear(sinfo, XGS_DMA_TX_CHAN); 4677 } else { 4678 xgsm_dma_chain_clear(sinfo, XGS_DMA_TX_CHAN); 4679 } 4680 tx_dcbs_done = bkn_do_tx(sinfo); 4681 bkn_tx_chain_done(sinfo, tx_dcbs_done); 4682 } 4683 4684 return sinfo->poll_channels ? budget : rx_dcbs_done; 4685 } 4686 4687 static int 4688 xgsx_do_dma(bkn_switch_info_t *sinfo, int budget) 4689 { 4690 int rx_dcbs_done = 0, tx_dcbs_done = 0; 4691 int chan_done, budget_chans = 0; 4692 uint32_t irq_stat, tx_dma_stat, rx_dma_stat[NUM_CMICX_RX_CHAN]; 4693 int chan; 4694 4695 DEV_READ32(sinfo, CMICX_IRQ_STATr, &irq_stat); 4696 DEV_READ32(sinfo, CMICX_DMA_STATr + 0x80 * XGS_DMA_TX_CHAN, &tx_dma_stat); 4697 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4698 DEV_READ32(sinfo, 4699 CMICX_DMA_STATr + 0x80 * (XGS_DMA_RX_CHAN + chan), 4700 &rx_dma_stat[chan]); 4701 } 4702 4703 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4704 if ((irq_stat & CMICX_DS_CMC_CTRLD_INT(XGS_DMA_RX_CHAN + chan)) || 4705 (irq_stat & CMICX_DS_CMC_DESC_DONE(XGS_DMA_RX_CHAN + chan))) { 4706 xgsx_dma_desc_clear(sinfo, XGS_DMA_RX_CHAN + chan); 4707 sinfo->poll_channels |= 1 << chan; 4708 } 4709 } 4710 if (!sinfo->poll_channels) { 4711 sinfo->poll_channels = (uint32_t)(1 << sinfo->rx_chans) - 1; 4712 budget_chans = budget / sinfo->rx_chans; 4713 } else { 4714 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4715 if (1 << chan & sinfo->poll_channels) { 4716 budget_chans++; 4717 } 4718 } 4719 budget_chans = budget / budget_chans; 4720 } 4721 4722 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4723 if (1 << chan & sinfo->poll_channels) { 4724 chan_done = bkn_do_rx(sinfo, chan, budget_chans); 4725 rx_dcbs_done += chan_done; 4726 if (chan_done < budget_chans) { 4727 sinfo->poll_channels &= ~(1 << chan); 4728 } 4729 bkn_rx_desc_done(sinfo, chan); 4730 } 4731 4732 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 4733 continue; 4734 } 4735 4736 if (rx_dma_stat[chan] & CMICX_DS_CMC_DMA_CHAIN_DONE) { 4737 xgsx_dma_chain_clear(sinfo, XGS_DMA_RX_CHAN + chan); 4738 bkn_rx_chain_done(sinfo, chan); 4739 } 4740 } 4741 4742 if ((irq_stat & CMICX_DS_CMC_CTRLD_INT(XGS_DMA_TX_CHAN)) || 4743 (tx_dma_stat & CMICX_DS_CMC_DMA_CHAIN_DONE)) { 4744 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN)) { 4745 xgsx_dma_desc_clear(sinfo, XGS_DMA_TX_CHAN); 4746 } else { 4747 xgsx_dma_chain_clear(sinfo, XGS_DMA_TX_CHAN); 4748 } 4749 tx_dcbs_done = bkn_do_tx(sinfo); 4750 bkn_tx_chain_done(sinfo, tx_dcbs_done); 4751 } 4752 4753 return sinfo->poll_channels ? budget : rx_dcbs_done; 4754 } 4755 4756 static int 4757 dev_do_dma(bkn_switch_info_t *sinfo, int budget) 4758 { 4759 if (DEV_IS_CMICX(sinfo)) { 4760 return xgsx_do_dma(sinfo, budget); 4761 } else if (DEV_IS_CMICM(sinfo)) { 4762 return xgsm_do_dma(sinfo, budget); 4763 } else { 4764 return xgs_do_dma(sinfo, budget); 4765 } 4766 } 4767 4768 static void 4769 xgs_isr(bkn_switch_info_t *sinfo) 4770 { 4771 uint32_t irq_stat; 4772 int rx_dcbs_done; 4773 4774 DEV_READ32(sinfo, CMIC_IRQ_STATr, &irq_stat); 4775 if ((irq_stat & sinfo->irq_mask) == 0) { 4776 /* Not ours */ 4777 return; 4778 } 4779 sinfo->interrupts++; 4780 4781 DBG_IRQ(("Got interrupt on device %d (0x%08x)\n", 4782 sinfo->dev_no, irq_stat)); 4783 4784 if (use_napi) { 4785 bkn_schedule_napi_poll(sinfo); 4786 } else { 4787 xgs_irq_mask_set(sinfo, 0); 4788 do { 4789 rx_dcbs_done = xgs_do_dma(sinfo, MAX_RX_DCBS); 4790 } while (rx_dcbs_done); 4791 } 4792 4793 xgs_irq_mask_set(sinfo, sinfo->irq_mask); 4794 } 4795 4796 static void 4797 xgsm_isr(bkn_switch_info_t *sinfo) 4798 { 4799 uint32_t irq_stat; 4800 int rx_dcbs_done; 4801 4802 DEV_READ32(sinfo, CMICM_IRQ_STATr, &irq_stat); 4803 if ((irq_stat & sinfo->irq_mask) == 0) { 4804 /* Not ours */ 4805 return; 4806 } 4807 sinfo->interrupts++; 4808 4809 DBG_IRQ(("Got interrupt on device %d (0x%08x)\n", 4810 sinfo->dev_no, irq_stat)); 4811 4812 if (use_napi) { 4813 bkn_schedule_napi_poll(sinfo); 4814 } else { 4815 xgsm_irq_mask_set(sinfo, 0); 4816 do { 4817 rx_dcbs_done = xgsm_do_dma(sinfo, MAX_RX_DCBS); 4818 if (sinfo->cdma_channels) { 4819 if (rx_dcbs_done >= MAX_RX_DCBS || sinfo->tx_yield) { 4820 /* Continuous DMA mode requires to yield timely */ 4821 break; 4822 } 4823 } 4824 } while (rx_dcbs_done); 4825 } 4826 4827 xgsm_irq_mask_set(sinfo, sinfo->irq_mask); 4828 } 4829 4830 static void 4831 xgsx_isr(bkn_switch_info_t *sinfo) 4832 { 4833 uint32_t irq_stat; 4834 int rx_dcbs_done; 4835 4836 DEV_READ32(sinfo, CMICX_IRQ_STATr, &irq_stat); 4837 if ((irq_stat & sinfo->irq_mask) == 0) { 4838 /* Not ours */ 4839 return; 4840 } 4841 sinfo->interrupts++; 4842 4843 DBG_IRQ(("Got interrupt on device %d (0x%08x)\n", 4844 sinfo->dev_no, irq_stat)); 4845 4846 if (use_napi) { 4847 bkn_schedule_napi_poll(sinfo); 4848 } else { 4849 xgsx_irq_mask_set(sinfo, 0); 4850 do { 4851 rx_dcbs_done = xgsx_do_dma(sinfo, MAX_RX_DCBS); 4852 if (sinfo->cdma_channels) { 4853 if (rx_dcbs_done >= MAX_RX_DCBS || sinfo->tx_yield) { 4854 /* Continuous DMA mode requires to yield timely */ 4855 break; 4856 } 4857 } 4858 } while (rx_dcbs_done); 4859 } 4860 4861 xgsx_irq_mask_set(sinfo, sinfo->irq_mask); 4862 } 4863 4864 static void 4865 bkn_isr(void *isr_data) 4866 { 4867 bkn_switch_info_t *sinfo = isr_data; 4868 4869 /* Safe exit on SMP systems */ 4870 if (!module_initialized) { 4871 return; 4872 } 4873 4874 /* Ensure that we do not touch registers during device reset */ 4875 if (sinfo->irq_mask == 0) { 4876 /* Not ours */ 4877 return; 4878 } 4879 4880 spin_lock(&sinfo->lock); 4881 4882 if (sinfo->napi_poll_mode) { 4883 /* Not ours */ 4884 spin_unlock(&sinfo->lock); 4885 return; 4886 } 4887 4888 if (DEV_IS_CMICX(sinfo)) { 4889 xgsx_isr(sinfo); 4890 } else if (DEV_IS_CMICM(sinfo)) { 4891 xgsm_isr(sinfo); 4892 } else { 4893 xgs_isr(sinfo); 4894 } 4895 4896 spin_unlock(&sinfo->lock); 4897 } 4898 4899 #ifdef CONFIG_NET_POLL_CONTROLLER 4900 static void 4901 bkn_poll_controller(struct net_device *dev) 4902 { 4903 bkn_priv_t *priv = netdev_priv(dev); 4904 4905 disable_irq(dev->irq); 4906 bkn_isr(priv->sinfo); 4907 enable_irq(dev->irq); 4908 } 4909 #endif 4910 4911 static void 4912 bkn_resume_rx(bkn_switch_info_t *sinfo) 4913 { 4914 bkn_desc_info_t *desc; 4915 bkn_dcb_chain_t *dcb_chain; 4916 uint64_t cur_halt, last_dcb, dcb_dma; 4917 int woffset, chan, cdma_running; 4918 4919 /* Resume Rx DMA on all channels */ 4920 for (chan = 0; chan < sinfo->rx_chans; chan++) { 4921 if (sinfo->rx[chan].use_rx_skb) { 4922 cdma_running = 0; 4923 bkn_api_rx_restart(sinfo, chan); 4924 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 4925 cur_halt = sinfo->halt_addr[XGS_DMA_RX_CHAN + chan]; 4926 last_dcb = sinfo->rx[chan].desc[MAX_RX_DCBS].dcb_dma; 4927 if (cur_halt != last_dcb) { 4928 desc = &sinfo->rx[chan].desc[sinfo->rx[chan].dirty + 1]; 4929 bkn_cdma_goto(sinfo, XGS_DMA_RX_CHAN + chan, desc->dcb_dma); 4930 cdma_running = 1; 4931 } 4932 } 4933 if (!cdma_running) { 4934 bkn_rx_restart(sinfo, chan); 4935 } 4936 } else { 4937 cdma_running = 0; 4938 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 4939 if (sinfo->rx[chan].api_active) { 4940 dcb_chain = sinfo->rx[chan].api_dcb_chain_end; 4941 woffset = (dcb_chain->dcb_cnt - 1) * sinfo->dcb_wsize; 4942 dcb_dma = dcb_chain->dcb_dma + woffset * sizeof(uint32_t); 4943 bkn_cdma_goto(sinfo, XGS_DMA_RX_CHAN + chan, dcb_dma); 4944 cdma_running = 1; 4945 } 4946 } 4947 if (!cdma_running) { 4948 bkn_api_rx_restart(sinfo, chan); 4949 } 4950 } 4951 } 4952 } 4953 4954 static int 4955 bkn_open(struct net_device *dev) 4956 { 4957 bkn_priv_t *priv = netdev_priv(dev); 4958 bkn_switch_info_t *sinfo = priv->sinfo; 4959 unsigned long flags; 4960 4961 /* Check if base device */ 4962 if (priv->id <= 0) { 4963 /* NAPI used only on base device */ 4964 if (use_napi) { 4965 bkn_napi_enable(dev, &sinfo->napi); 4966 } 4967 4968 /* Start DMA when base device is started */ 4969 if (sinfo->basedev_suspended) { 4970 spin_lock_irqsave(&sinfo->lock, flags); 4971 dev_do_dma(sinfo, MAX_RX_DCBS); 4972 sinfo->basedev_suspended = 0; 4973 bkn_api_tx(sinfo); 4974 if (!sinfo->tx.api_active) { 4975 bkn_resume_tx(sinfo); 4976 } 4977 bkn_resume_rx(sinfo); 4978 spin_unlock_irqrestore(&sinfo->lock, flags); 4979 } 4980 } 4981 4982 if (!sinfo->basedev_suspended) { 4983 netif_start_queue(dev); 4984 } 4985 4986 return 0; 4987 } 4988 4989 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29)) 4990 static int 4991 bkn_set_mac_address(struct net_device *dev, void *addr) 4992 { 4993 if (!is_valid_ether_addr(((struct sockaddr *)addr)->sa_data)) { 4994 return -EINVAL; 4995 } 4996 memcpy(dev->dev_addr, ((struct sockaddr *)addr)->sa_data, dev->addr_len); 4997 return 0; 4998 } 4999 #endif 5000 5001 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29)) 5002 static int 5003 bkn_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 5004 { 5005 bkn_priv_t *priv = netdev_priv(dev); 5006 bkn_switch_info_t *sinfo = priv->sinfo; 5007 struct hwtstamp_config config; 5008 5009 if (cmd == SIOCSHWTSTAMP) { 5010 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) { 5011 return -EFAULT; 5012 } 5013 5014 if (!knet_hw_tstamp_enable_cb || !knet_hw_tstamp_disable_cb || 5015 priv->type != KCOM_NETIF_T_PORT) { 5016 return -ERANGE; 5017 } 5018 5019 switch (config.tx_type) { 5020 case HWTSTAMP_TX_OFF: 5021 knet_hw_tstamp_disable_cb(sinfo->dev_no, priv->port); 5022 sinfo->tx_hwts = 0; 5023 break; 5024 case HWTSTAMP_TX_ON: 5025 knet_hw_tstamp_enable_cb(sinfo->dev_no, priv->port); 5026 sinfo->tx_hwts = 1; 5027 break; 5028 default: 5029 return -ERANGE; 5030 } 5031 5032 switch (config.rx_filter) { 5033 case HWTSTAMP_FILTER_NONE: 5034 if (sinfo->rx_hwts) { 5035 knet_hw_tstamp_disable_cb(sinfo->dev_no, priv->port); 5036 sinfo->rx_hwts = 0; 5037 } 5038 break; 5039 default: 5040 if (!sinfo->rx_hwts) { 5041 knet_hw_tstamp_enable_cb(sinfo->dev_no, priv->port); 5042 sinfo->rx_hwts = 1; 5043 } 5044 config.rx_filter = HWTSTAMP_FILTER_ALL; 5045 break; 5046 } 5047 5048 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? -EFAULT : 0; 5049 } 5050 5051 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0)) 5052 if (cmd == SIOCGHWTSTAMP) { 5053 config.flags = 0; 5054 config.tx_type = sinfo->tx_hwts ? HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF; 5055 config.rx_filter = sinfo->rx_hwts ? HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE; 5056 5057 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? -EFAULT : 0; 5058 } 5059 #endif 5060 5061 return -EINVAL; 5062 } 5063 #endif 5064 5065 static int 5066 bkn_change_mtu(struct net_device *dev, int new_mtu) 5067 { 5068 int max_size = new_mtu + ETH_HLEN + VLAN_HLEN + 4; 5069 5070 if (new_mtu < 68 || max_size > rx_buffer_size) { 5071 return -EINVAL; 5072 } 5073 dev->mtu = new_mtu; 5074 return 0; 5075 } 5076 5077 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) 5078 static int 5079 bkn_poll(struct net_device *dev, int *budget) 5080 { 5081 bkn_priv_t *priv = netdev_priv(dev); 5082 bkn_switch_info_t *sinfo = priv->sinfo; 5083 int cur_budget = *budget; 5084 int poll_again = 0; 5085 int rx_dcbs_done; 5086 unsigned long flags; 5087 5088 spin_lock_irqsave(&sinfo->lock, flags); 5089 5090 DBG_NAPI(("NAPI poll on %s.\n", dev->name)); 5091 5092 sinfo->napi_poll_again = 0; 5093 5094 if (cur_budget > dev->quota) { 5095 cur_budget = dev->quota; 5096 } 5097 5098 rx_dcbs_done = dev_do_dma(sinfo, cur_budget); 5099 5100 *budget -= rx_dcbs_done; 5101 cur_budget -= rx_dcbs_done; 5102 dev->quota -= rx_dcbs_done; 5103 5104 if (sinfo->napi_poll_again || cur_budget <= 0) { 5105 poll_again = 1; 5106 sinfo->napi_not_done++; 5107 } else { 5108 bkn_napi_poll_complete(sinfo); 5109 } 5110 5111 spin_unlock_irqrestore(&sinfo->lock, flags); 5112 5113 return poll_again; 5114 } 5115 #else 5116 static int 5117 bkn_poll(struct napi_struct *napi, int budget) 5118 { 5119 bkn_switch_info_t *sinfo = container_of(napi, bkn_switch_info_t, napi); 5120 int rx_dcbs_done; 5121 unsigned long flags; 5122 5123 spin_lock_irqsave(&sinfo->lock, flags); 5124 5125 DBG_NAPI(("NAPI poll on %s.\n", sinfo->dev->name)); 5126 5127 sinfo->napi_poll_again = 0; 5128 5129 rx_dcbs_done = dev_do_dma(sinfo, budget); 5130 5131 if (sinfo->napi_poll_again || rx_dcbs_done >= budget) { 5132 /* Force poll again */ 5133 rx_dcbs_done = budget; 5134 sinfo->napi_not_done++; 5135 } else { 5136 bkn_napi_poll_complete(sinfo); 5137 } 5138 5139 spin_unlock_irqrestore(&sinfo->lock, flags); 5140 5141 return rx_dcbs_done; 5142 } 5143 #endif 5144 5145 static int 5146 bkn_stop(struct net_device *dev) 5147 { 5148 bkn_priv_t *priv = netdev_priv(dev); 5149 bkn_switch_info_t *sinfo = priv->sinfo; 5150 unsigned long flags; 5151 5152 netif_stop_queue(dev); 5153 5154 /* Check if base device */ 5155 if (priv->id <= 0) { 5156 /* NAPI used only on base device */ 5157 if (use_napi) { 5158 bkn_napi_disable(dev, &sinfo->napi); 5159 } 5160 /* Suspend all devices if base device is stopped */ 5161 if (basedev_suspend) { 5162 spin_lock_irqsave(&sinfo->lock, flags); 5163 bkn_suspend_tx(sinfo); 5164 sinfo->basedev_suspended = 1; 5165 spin_unlock_irqrestore(&sinfo->lock, flags); 5166 } 5167 } 5168 5169 return 0; 5170 } 5171 5172 /* 5173 * Network Device Statistics. 5174 * Cleared at init time. 5175 */ 5176 static struct net_device_stats * 5177 bkn_get_stats(struct net_device *dev) 5178 { 5179 bkn_priv_t *priv = netdev_priv(dev); 5180 5181 return &priv->stats; 5182 } 5183 5184 /* Fake multicast ability */ 5185 static void 5186 bkn_set_multicast_list(struct net_device *dev) 5187 { 5188 } 5189 5190 static int 5191 bkn_hw_tstamp_tx_config(bkn_switch_info_t *sinfo, struct sk_buff *skb, uint32_t *meta) 5192 { 5193 uint32_t *md = NULL; 5194 5195 if (!knet_hw_tstamp_tx_meta_get_cb) { 5196 return -1; 5197 } 5198 5199 KNET_SKB_CB(skb)->dcb_type = sinfo->dcb_type & 0xFFFF; 5200 knet_hw_tstamp_tx_meta_get_cb(sinfo->dev_no, skb, &md); 5201 if (!md) { 5202 return -1; 5203 } 5204 5205 switch (sinfo->dcb_type) { 5206 case 26: 5207 case 32: 5208 case 33: 5209 meta[2] |= md[0]; 5210 meta[3] |= md[1]; 5211 meta[4] |= md[2]; 5212 meta[5] |= md[3]; 5213 break; 5214 case 36: 5215 case 38: 5216 meta[0] |= md[0]; 5217 meta[1] |= md[1]; 5218 meta[2] |= md[2]; 5219 meta[3] |= md[3]; 5220 break; 5221 default: 5222 return -1; 5223 } 5224 5225 return 0; 5226 } 5227 5228 static int 5229 bkn_tx(struct sk_buff *skb, struct net_device *dev) 5230 { 5231 bkn_priv_t *priv = netdev_priv(dev); 5232 bkn_switch_info_t *sinfo = priv->sinfo; 5233 struct sk_buff *new_skb = NULL; 5234 unsigned char *pktdata; 5235 int pktlen, hdrlen, taglen, rcpulen, metalen; 5236 int sop, idx; 5237 uint16_t tpid; 5238 uint32_t *metadata; 5239 unsigned long flags; 5240 5241 DBG_VERB(("Netif Tx: Len=%d\n", skb->len)); 5242 5243 if (priv->id <= 0) { 5244 /* Do not transmit on base device */ 5245 priv->stats.tx_dropped++; 5246 dev_kfree_skb_any(skb); 5247 return 0; 5248 } 5249 5250 if (!netif_carrier_ok(dev)) { 5251 DBG_WARN(("Tx drop: Invalid RCPU encapsulation\n")); 5252 priv->stats.tx_dropped++; 5253 sinfo->tx.pkts_d_no_link++; 5254 dev_kfree_skb_any(skb); 5255 return 0; 5256 } 5257 5258 spin_lock_irqsave(&sinfo->lock, flags); 5259 5260 if (sinfo->tx.free > 1) { 5261 bkn_desc_info_t *desc = &sinfo->tx.desc[sinfo->tx.cur]; 5262 uint32_t *dcb, *meta; 5263 5264 pktdata = skb->data; 5265 pktlen = skb->len; 5266 hdrlen = (sinfo->cmic_type == 'x' ) ? ((device_is_dnx(sinfo)) ? priv->system_headers_size: PKT_TX_HDR_SIZE) : 0; 5267 rcpulen = 0; 5268 sop = 0; 5269 5270 if (priv->flags & KCOM_NETIF_F_RCPU_ENCAP) { 5271 rcpulen = RCPU_HDR_SIZE; 5272 if (skb->len < (rcpulen + 14)) { 5273 DBG_WARN(("Tx drop: Invalid RCPU encapsulation\n")); 5274 priv->stats.tx_dropped++; 5275 sinfo->tx.pkts_d_rcpu_encap++; 5276 dev_kfree_skb_any(skb); 5277 spin_unlock_irqrestore(&sinfo->lock, flags); 5278 return 0; 5279 } 5280 if (check_rcpu_signature && 5281 ((skb->data[18] << 8) | skb->data[19]) != sinfo->rcpu_sig) { 5282 DBG_WARN(("Tx drop: Invalid RCPU signature\n")); 5283 priv->stats.tx_dropped++; 5284 sinfo->tx.pkts_d_rcpu_sig++; 5285 dev_kfree_skb_any(skb); 5286 spin_unlock_irqrestore(&sinfo->lock, flags); 5287 return 0; 5288 } 5289 if (skb->data[21] & RCPU_F_MODHDR) { 5290 sop = skb->data[RCPU_HDR_SIZE]; 5291 switch (sop) { 5292 case 0xff: 5293 case 0x81: 5294 case 0xfb: 5295 case 0xfc: 5296 break; 5297 default: 5298 DBG_WARN(("Tx drop: Invalid RCPU meta data\n")); 5299 priv->stats.tx_dropped++; 5300 sinfo->tx.pkts_d_rcpu_meta++; 5301 dev_kfree_skb_any(skb); 5302 spin_unlock_irqrestore(&sinfo->lock, flags); 5303 return 0; 5304 } 5305 if (sinfo->cmic_type != 'x') { 5306 rcpulen += RCPU_TX_META_SIZE; 5307 } 5308 } 5309 /* Skip over RCPU encapsulation */ 5310 pktdata = &skb->data[rcpulen]; 5311 pktlen -= rcpulen; 5312 5313 /* CPU packets require tag */ 5314 if (sop == 0) { 5315 hdrlen = 0; 5316 tpid = (pktdata[12] << 8) | pktdata[13]; 5317 if (tpid != 0x8100) { 5318 if (skb_header_cloned(skb)) { 5319 /* Current SKB cannot be modified */ 5320 DBG_SKB(("Realloc Tx SKB\n")); 5321 new_skb = dev_alloc_skb(pktlen + TAG_SZ + FCS_SZ); 5322 if (new_skb == NULL) { 5323 DBG_WARN(("Tx drop: No SKB memory\n")); 5324 priv->stats.tx_dropped++; 5325 sinfo->tx.pkts_d_no_skb++; 5326 dev_kfree_skb_any(skb); 5327 spin_unlock_irqrestore(&sinfo->lock, flags); 5328 return 0; 5329 } 5330 memcpy(new_skb->data, pktdata, 12); 5331 memcpy(&new_skb->data[16], &pktdata[12], pktlen - 12); 5332 skb_put(new_skb, pktlen + TAG_SZ); 5333 dev_kfree_skb_any(skb); 5334 skb = new_skb; 5335 pktdata = skb->data; 5336 rcpulen = 0; 5337 } else { 5338 /* Add tag to RCPU header space */ 5339 DBG_SKB(("Expand into unused RCPU header\n")); 5340 rcpulen -= TAG_SZ; 5341 pktdata = &skb->data[rcpulen]; 5342 for (idx = 0; idx < 12; idx++) { 5343 pktdata[idx] = pktdata[idx + TAG_SZ]; 5344 } 5345 } 5346 pktdata[12] = 0x81; 5347 pktdata[13] = 0x00; 5348 pktdata[14] = (priv->vlan >> 8) & 0xf; 5349 pktdata[15] = priv->vlan & 0xff; 5350 pktlen += TAG_SZ; 5351 } 5352 } 5353 } else { 5354 if (sinfo->cmic_type == 'x' && priv->port >= 0) { 5355 if (skb_header_cloned(skb) || skb_headroom(skb) < hdrlen + 4) { 5356 /* Current SKB cannot be modified */ 5357 DBG_SKB(("Realloc Tx SKB\n")); 5358 new_skb = dev_alloc_skb(pktlen + hdrlen + 4 + FCS_SZ); 5359 if (new_skb == NULL) { 5360 DBG_WARN(("Tx drop: No SKB memory\n")); 5361 priv->stats.tx_dropped++; 5362 sinfo->tx.pkts_d_no_skb++; 5363 dev_kfree_skb_any(skb); 5364 spin_unlock_irqrestore(&sinfo->lock, flags); 5365 return 0; 5366 } 5367 if (!device_is_dnx(sinfo)) 5368 { 5369 skb_reserve(new_skb, 4); 5370 } 5371 memcpy(new_skb->data + hdrlen, skb->data, pktlen); 5372 skb_put(new_skb, pktlen + hdrlen); 5373 dev_kfree_skb_any(skb); 5374 skb = new_skb; 5375 } else { 5376 DBG_SKB(("Expand Tx SKB\n")); 5377 skb_push(skb, hdrlen); 5378 } 5379 memset(skb->data, 0, hdrlen); 5380 pktdata = skb->data; 5381 pktlen += hdrlen; 5382 } else { 5383 hdrlen = 0; 5384 } 5385 5386 if (priv->port < 0 || (priv->flags & KCOM_NETIF_F_ADD_TAG)) { 5387 DBG_DUNE(("ADD VLAN TAG\n")); 5388 /* Need to add VLAN tag if packet is untagged */ 5389 tpid = (skb->data[hdrlen + 12] << 8) | skb->data[hdrlen + 13]; 5390 if (tpid != 0x8100) { 5391 if (skb_header_cloned(skb) || skb_headroom(skb) < 4) { 5392 /* Current SKB cannot be modified */ 5393 DBG_SKB(("Realloc Tx SKB\n")); 5394 new_skb = dev_alloc_skb(pktlen + TAG_SZ + FCS_SZ); 5395 if (new_skb == NULL) { 5396 DBG_WARN(("Tx drop: No SKB memory\n")); 5397 priv->stats.tx_dropped++; 5398 sinfo->tx.pkts_d_no_skb++; 5399 dev_kfree_skb_any(skb); 5400 spin_unlock_irqrestore(&sinfo->lock, flags); 5401 return 0; 5402 } 5403 memcpy(new_skb->data, skb->data, hdrlen + 12); 5404 memcpy(&new_skb->data[hdrlen + 16], &skb->data[hdrlen + 12], 5405 pktlen - hdrlen - 12); 5406 skb_put(new_skb, pktlen + TAG_SZ); 5407 dev_kfree_skb_any(skb); 5408 skb = new_skb; 5409 } else { 5410 /* Add tag to existing buffer */ 5411 DBG_SKB(("Expand Tx SKB\n")); 5412 skb_push(skb, TAG_SZ); 5413 for (idx = 0; idx < hdrlen + 12; idx++) { 5414 skb->data[idx] = skb->data[idx + TAG_SZ]; 5415 } 5416 } 5417 pktdata = skb->data; 5418 pktdata[hdrlen + 12] = 0x81; 5419 pktdata[hdrlen + 13] = 0x00; 5420 pktdata[hdrlen + 14] = (priv->vlan >> 8) & 0xf; 5421 pktdata[hdrlen + 15] = priv->vlan & 0xff; 5422 pktlen += TAG_SZ; 5423 } 5424 } 5425 } 5426 5427 /* Pad packet if needed */ 5428 taglen = 0; 5429 tpid = (pktdata[hdrlen + 12] << 8) | pktdata[hdrlen + 13]; 5430 if (tpid == 0x8100) { 5431 taglen = 4; 5432 } 5433 if (pktlen < (60 + taglen + hdrlen)) { 5434 pktlen = (60 + taglen + hdrlen); 5435 if (SKB_PADTO(skb, pktlen) != 0) { 5436 DBG_WARN(("Tx drop: skb_padto failed\n")); 5437 priv->stats.tx_dropped++; 5438 sinfo->tx.pkts_d_pad_fail++; 5439 dev_kfree_skb_any(skb); 5440 spin_unlock_irqrestore(&sinfo->lock, flags); 5441 return 0; 5442 } 5443 /* skb_padto may update the skb->data pointer */ 5444 pktdata = &skb->data[rcpulen]; 5445 } 5446 5447 if ((pktlen + FCS_SZ) > SOC_DCB_KNET_COUNT_MASK) { 5448 DBG_WARN(("Tx drop: size of pkt (%d) is out of range(%d)\n", 5449 (pktlen + FCS_SZ), SOC_DCB_KNET_COUNT_MASK)); 5450 sinfo->tx.pkts_d_over_limit++; 5451 priv->stats.tx_dropped++; 5452 dev_kfree_skb_any(skb); 5453 spin_unlock_irqrestore(&sinfo->lock, flags); 5454 return 0; 5455 } 5456 5457 dcb = desc->dcb_mem; 5458 meta = (sinfo->cmic_type == 'x') ? (uint32_t *)pktdata : dcb; 5459 memset(dcb, 0, sinfo->dcb_wsize * sizeof(uint32_t)); 5460 if (priv->flags & KCOM_NETIF_F_RCPU_ENCAP) { 5461 /* If module header SOP is non-zero, use RCPU meta data */ 5462 if (sop != 0) { 5463 if (sinfo->cmic_type == 'x') { 5464 dcb[2] |= 1 << 19; 5465 } else { 5466 metalen = (sinfo->dcb_wsize - 3) * sizeof(uint32_t); 5467 if (metalen > RCPU_TX_META_SIZE) { 5468 metalen = RCPU_TX_META_SIZE; 5469 } 5470 metadata = (uint32_t *)&skb->data[RCPU_HDR_SIZE]; 5471 for (idx = 0; idx < BYTES2WORDS(metalen); idx++) { 5472 dcb[idx + 2] = ntohl(metadata[idx]); 5473 } 5474 dcb[1] |= 1 << 19; 5475 } 5476 } 5477 } else if (priv->port >= 0) { 5478 /* Send to physical port */ 5479 if (sinfo->cmic_type == 'x') { 5480 dcb[2] |= 1 << 19; 5481 } else { 5482 dcb[1] |= 1 << 19; 5483 } 5484 switch (sinfo->dcb_type) { 5485 case 23: 5486 case 26: 5487 case 30: 5488 case 31: 5489 case 34: 5490 case 37: 5491 dcb[2] = 0x81000000; 5492 dcb[3] = priv->port; 5493 dcb[3] |= (priv->qnum & 0xc00) << 20; 5494 dcb[4] = 0x00040000; 5495 dcb[4] |= (priv->qnum & 0x3ff) << 8; 5496 break; 5497 case 24: 5498 dcb[2] = 0xff000000; 5499 dcb[3] = 0x00000100; 5500 dcb[4] = priv->port; 5501 dcb[4] |= (priv->qnum & 0xfff) << 14; 5502 break; 5503 case 28: 5504 { 5505 if (priv->type == KCOM_NETIF_T_PORT) { 5506 /* add PTCH ITMH header */ 5507 if (skb_header_cloned(skb) || skb_headroom(skb) < 4) { 5508 /* Current SKB cannot be modified */ 5509 DBG_SKB(("Realloc Tx SKB for DNX ITMH header\n")); 5510 new_skb = dev_alloc_skb(pktlen + 4 + 2 + FCS_SZ); 5511 if (new_skb == NULL) { 5512 DBG_WARN(("Tx drop: No SKB memory for DNX ITMH header\n")); 5513 priv->stats.tx_dropped++; 5514 sinfo->tx.pkts_d_no_skb++; 5515 dev_kfree_skb_any(skb); 5516 spin_unlock_irqrestore(&sinfo->lock, flags); 5517 return 0; 5518 } 5519 memcpy(&new_skb->data[6], skb->data, pktlen); 5520 skb_put(new_skb, pktlen + 6); 5521 dev_kfree_skb_any(skb); 5522 skb = new_skb; 5523 } else { 5524 /* Add tag to existing buffer */ 5525 DBG_SKB(("Expand Tx SKB for DNX ITMH header\n")); 5526 skb_push(skb, 6); 5527 } 5528 pktdata = skb->data; 5529 pktdata[0] = 0x50; 5530 pktdata[1] = 0x00; 5531 memcpy(&pktdata[2], priv->itmh, 4); 5532 pktlen += 6; 5533 } 5534 else if (priv->type == KCOM_NETIF_T_VLAN) { 5535 /* add PTCH header */ 5536 if (skb_header_cloned(skb) || skb_headroom(skb) < 4) { 5537 /* Current SKB cannot be modified */ 5538 DBG_SKB(("Realloc Tx SKB for DNX header\n")); 5539 new_skb = dev_alloc_skb(pktlen + 2 + FCS_SZ); 5540 if (new_skb == NULL) { 5541 DBG_WARN(("Tx drop: No SKB memory for DNX header\n")); 5542 priv->stats.tx_dropped++; 5543 sinfo->tx.pkts_d_no_skb++; 5544 dev_kfree_skb_any(skb); 5545 spin_unlock_irqrestore(&sinfo->lock, flags); 5546 return 0; 5547 } 5548 memcpy(&new_skb->data[2], skb->data, pktlen); 5549 skb_put(new_skb, pktlen + 2); 5550 dev_kfree_skb_any(skb); 5551 skb = new_skb; 5552 } else { 5553 /* Add tag to existing buffer */ 5554 DBG_SKB(("Expand Tx SKB for DNX header\n")); 5555 skb_push(skb, 2); 5556 } 5557 pktdata = skb->data; 5558 pktdata[0] = 0xd0; 5559 pktdata[1] = priv->port; 5560 pktlen += 2; 5561 } 5562 dcb[1] = pktlen; 5563 break; 5564 } 5565 case 29: 5566 dcb[2] = 0x81000000; 5567 dcb[3] = priv->port; 5568 dcb[4] = 0x00100000; 5569 dcb[4] |= (priv->qnum & 0xfff) << 8; 5570 break; 5571 case 32: 5572 dcb[2] = 0x81000000; 5573 dcb[3] = priv->port; 5574 dcb[4] = 0x00004000; 5575 dcb[4] |= (priv->qnum & 0x3f) << 8; 5576 break; 5577 case 33: 5578 dcb[2] = 0x81000000; 5579 dcb[3] = (priv->port) << 2; 5580 dcb[4] = 0x00100000; 5581 dcb[4] |= (priv->qnum & 0xfff) << 8; 5582 break; 5583 case 35: 5584 dcb[2] = 0x81000000; 5585 dcb[3] = (priv->port) << 4; 5586 dcb[4] = 0x00400000; 5587 dcb[4] |= (priv->qnum & 0x3fff) << 8; 5588 break; 5589 case 36: 5590 if (sinfo->cmic_type == 'x') { 5591 meta[0] = htonl(0x81000000); 5592 meta[1] = htonl(priv->port); 5593 meta[2] = htonl(0x00008000 | (priv->qnum & 0x3f) << 9); 5594 } else { 5595 dcb[2] = 0x81000000; 5596 dcb[3] = priv->port; 5597 dcb[4] = 0x00008000; 5598 dcb[4] |= (priv->qnum & 0x3f) << 9; 5599 } 5600 break; 5601 case 38: 5602 if (sinfo->cmic_type == 'x') { 5603 meta[0] = htonl(0x81000000); 5604 meta[1] = htonl(priv->port); 5605 meta[2] = htonl(0x00004000 | (priv->qnum & 0x3f) << 8); 5606 } else { 5607 dcb[2] = 0x81000000; 5608 dcb[3] = priv->port; 5609 dcb[4] = 0x00004000; 5610 dcb[4] |= (priv->qnum & 0x3f) << 8; 5611 } 5612 break; 5613 case 39: 5614 if (device_is_dnx(sinfo)) { 5615 /* 5616 * if KCOM_NETIF_T_PORT, add MH+PTCH+ITMH header 5617 * if KCOM_NETIF_T_VLAN, add MH+PTCH+header 5618 */ 5619 pktdata = skb->data; 5620 memcpy(&pktdata[0], priv->system_headers, priv->system_headers_size); 5621 } 5622 break; 5623 default: 5624 dcb[2] = 0xff000000; 5625 dcb[3] = 0x00000100; 5626 dcb[4] = priv->port; 5627 break; 5628 } 5629 } 5630 5631 /* Optional SKB updates */ 5632 if (knet_tx_cb != NULL) { 5633 KNET_SKB_CB(skb)->netif_user_data = priv->cb_user_data; 5634 KNET_SKB_CB(skb)->dcb_type = sinfo->dcb_type & 0xFFFF; 5635 skb = knet_tx_cb(skb, sinfo->dev_no, meta); 5636 if (skb == NULL) { 5637 /* Consumed by call-back */ 5638 DBG_WARN(("Tx drop: Consumed by call-back\n")); 5639 priv->stats.tx_dropped++; 5640 sinfo->tx.pkts_d_callback++; 5641 spin_unlock_irqrestore(&sinfo->lock, flags); 5642 return 0; 5643 } 5644 /* Restore (possibly) altered packet variables 5645 * bit0 -bit15 of dcb[1] is used to save requested byte count 5646 */ 5647 if ((skb->len + FCS_SZ) <= SOC_DCB_KNET_COUNT_MASK) { 5648 pktlen = skb->len; 5649 if (pktlen < (60 + taglen + hdrlen)) { 5650 pktlen = (60 + taglen + hdrlen); 5651 if (SKB_PADTO(skb, pktlen) != 0) { 5652 DBG_WARN(("Tx drop: skb_padto failed\n")); 5653 priv->stats.tx_dropped++; 5654 sinfo->tx.pkts_d_pad_fail++; 5655 dev_kfree_skb_any(skb); 5656 spin_unlock_irqrestore(&sinfo->lock, flags); 5657 return 0; 5658 } 5659 DBG_SKB(("Packet padded to %d bytes after tx callback\n", pktlen)); 5660 } 5661 pktdata = skb->data; 5662 if (sinfo->cmic_type == 'x') { 5663 meta = (uint32_t *)pktdata; 5664 } 5665 } else { 5666 DBG_WARN(("Tx drop: size of pkt (%d) is out of range(%d)\n", 5667 (pktlen + FCS_SZ), SOC_DCB_KNET_COUNT_MASK)); 5668 sinfo->tx.pkts_d_over_limit++; 5669 priv->stats.tx_dropped++; 5670 sinfo->tx.pkts_d_callback++; 5671 dev_kfree_skb_any(skb); 5672 spin_unlock_irqrestore(&sinfo->lock, flags); 5673 return 0; 5674 } 5675 } 5676 5677 /* Do Tx timestamping */ 5678 if (priv->port >= 0) { 5679 if (bkn_skb_tx_flags(skb) & SKBTX_HW_TSTAMP && sinfo->tx_hwts) { 5680 KNET_SKB_CB(skb)->port = priv->port; 5681 bkn_hw_tstamp_tx_config(sinfo, skb, meta); 5682 bkn_skb_tx_flags(skb) |= SKBTX_IN_PROGRESS; 5683 } 5684 bkn_skb_tx_timestamp(skb); 5685 } 5686 5687 /* Prepare for DMA */ 5688 desc->skb = skb; 5689 /* Add FCS bytes */ 5690 pktlen = pktlen + FCS_SZ; 5691 desc->dma_size = pktlen; 5692 desc->skb_dma = DMA_MAP_SINGLE(sinfo->dma_dev, 5693 pktdata, desc->dma_size, 5694 DMA_TODEV); 5695 if (DMA_MAPPING_ERROR(sinfo->dma_dev, desc->skb_dma)) { 5696 priv->stats.tx_dropped++; 5697 dev_kfree_skb_any(skb); 5698 spin_unlock_irqrestore(&sinfo->lock, flags); 5699 return 0; 5700 } 5701 dcb[0] = desc->skb_dma; 5702 if (sinfo->cmic_type == 'x') { 5703 dcb[1] = DMA_TO_BUS_HI(desc->skb_dma >> 32); 5704 dcb[2] &= ~SOC_DCB_KNET_COUNT_MASK; 5705 dcb[2] |= pktlen; 5706 } else { 5707 dcb[1] &= ~SOC_DCB_KNET_COUNT_MASK; 5708 dcb[1] |= pktlen; 5709 } 5710 5711 bkn_dump_dcb("Tx RCPU", dcb, sinfo->dcb_wsize, XGS_DMA_TX_CHAN); 5712 DBG_DCB_TX(("Add Tx DCB @ 0x%08x (%d) [%d free] (%d bytes).\n", 5713 (uint32_t)desc->dcb_dma, sinfo->tx.cur, 5714 sinfo->tx.free, pktlen)); 5715 bkn_dump_pkt(pktdata, pktlen, XGS_DMA_TX_CHAN); 5716 5717 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN)) { 5718 if (sinfo->cmic_type == 'x') { 5719 dcb[2] |= 1 << 24 | 1 << 16; 5720 } else { 5721 dcb[1] |= 1 << 24 | 1 << 16; 5722 } 5723 } else { 5724 bkn_tx_dma_start(sinfo); 5725 } 5726 if (++sinfo->tx.cur >= MAX_TX_DCBS) { 5727 sinfo->tx.cur = 0; 5728 } 5729 sinfo->tx.free--; 5730 5731 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN) && !sinfo->tx.api_active) { 5732 /* DMA run to the new halt location */ 5733 bkn_cdma_goto(sinfo, XGS_DMA_TX_CHAN, 5734 sinfo->tx.desc[sinfo->tx.cur].dcb_dma); 5735 } 5736 5737 priv->stats.tx_packets++; 5738 priv->stats.tx_bytes += pktlen; 5739 sinfo->tx.pkts++; 5740 } else { 5741 DBG_WARN(("Tx drop: No DMA resources\n")); 5742 priv->stats.tx_dropped++; 5743 sinfo->tx.pkts_d_dma_resrc++; 5744 dev_kfree_skb_any(skb); 5745 } 5746 5747 /* Check our Tx resources */ 5748 if (sinfo->tx.free <= 1) { 5749 bkn_suspend_tx(sinfo); 5750 } 5751 5752 NETDEV_UPDATE_TRANS_START_TIME(dev); 5753 5754 spin_unlock_irqrestore(&sinfo->lock, flags); 5755 5756 return 0; 5757 } 5758 5759 static void 5760 bkn_timer_func(bkn_switch_info_t *sinfo) 5761 { 5762 unsigned long flags; 5763 int chan; 5764 int restart_timer; 5765 5766 spin_lock_irqsave(&sinfo->lock, flags); 5767 5768 sinfo->timer_runs++; 5769 5770 restart_timer = 0; 5771 for (chan = 0; chan < sinfo->rx_chans; chan++) { 5772 /* Restart channel if not running */ 5773 if (sinfo->rx[chan].running == 0) { 5774 bkn_rx_refill(sinfo, chan); 5775 if (bkn_rx_restart(sinfo, chan) != 0) { 5776 restart_timer = 1; 5777 } 5778 } 5779 } 5780 5781 if (restart_timer) { 5782 /* Presumably still out of memory */ 5783 sinfo->timer.expires = jiffies + 1; 5784 add_timer(&sinfo->timer); 5785 } else { 5786 sinfo->timer_queued = 0; 5787 } 5788 5789 spin_unlock_irqrestore(&sinfo->lock, flags); 5790 } 5791 5792 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4,15,0)) 5793 static void 5794 bkn_timer(unsigned long context) 5795 { 5796 bkn_switch_info_t *sinfo = (bkn_switch_info_t *)context; 5797 return bkn_timer_func(sinfo); 5798 } 5799 #else 5800 static void 5801 bkn_timer(struct timer_list *t) 5802 { 5803 bkn_switch_info_t *sinfo = from_timer(sinfo, t, timer); 5804 return bkn_timer_func(sinfo); 5805 } 5806 #endif 5807 5808 static void 5809 bkn_rx_add_tokens(bkn_switch_info_t *sinfo, int chan) 5810 { 5811 unsigned long cur_jif, ticks; 5812 uint32_t tokens_per_tick; 5813 bkn_desc_info_t *desc; 5814 5815 tokens_per_tick = sinfo->rx[chan].rate_max / HZ; 5816 cur_jif = jiffies; 5817 ticks = cur_jif - sinfo->rx[chan].tok_jif; 5818 sinfo->rx[chan].tokens += ticks * tokens_per_tick; 5819 sinfo->rx[chan].tok_jif = cur_jif; 5820 if (sinfo->rx[chan].tokens > sinfo->rx[chan].burst_max) { 5821 sinfo->rx[chan].tokens = sinfo->rx[chan].burst_max; 5822 } 5823 5824 /* Restart channel if Rx is suppressed */ 5825 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 5826 bkn_rx_refill(sinfo, chan); 5827 desc = &sinfo->rx[chan].desc[sinfo->rx[chan].dirty]; 5828 if (desc->dcb_dma == sinfo->halt_addr[XGS_DMA_RX_CHAN + chan]) { 5829 desc = &sinfo->rx[chan].desc[MAX_RX_DCBS]; 5830 bkn_cdma_goto(sinfo, XGS_DMA_RX_CHAN + chan, desc->dcb_dma); 5831 } 5832 } else { 5833 if (sinfo->rx[chan].running == 0) { 5834 bkn_rx_refill(sinfo, chan); 5835 bkn_rx_restart(sinfo, chan); 5836 } 5837 } 5838 } 5839 5840 static void 5841 bkn_rxtick_func(bkn_switch_info_t *sinfo) 5842 { 5843 unsigned long flags; 5844 unsigned long cur_jif, ticks; 5845 uint32_t pkt_diff; 5846 int chan; 5847 5848 spin_lock_irqsave(&sinfo->lock, flags); 5849 5850 sinfo->rxtick.expires = jiffies + sinfo->rxtick_jiffies; 5851 5852 /* For debug purposes we maintain a rough actual packet rate */ 5853 if (++sinfo->rxticks >= sinfo->rxticks_per_sec) { 5854 for (chan = 0; chan < sinfo->rx_chans; chan++) { 5855 pkt_diff = sinfo->rx[chan].pkts - sinfo->rx[chan].pkts_ref; 5856 cur_jif = jiffies; 5857 ticks = cur_jif - sinfo->rx[chan].rate_jif; 5858 sinfo->rx[chan].rate = (pkt_diff * HZ) / ticks; 5859 sinfo->rx[chan].rate_jif = cur_jif; 5860 sinfo->rx[chan].pkts_ref = sinfo->rx[chan].pkts; 5861 } 5862 sinfo->rxticks = 0; 5863 } 5864 5865 /* Update tokens for Rx rate control */ 5866 for (chan = 0; chan < sinfo->rx_chans; chan++) { 5867 if (sinfo->rx[chan].tokens < sinfo->rx[chan].burst_max) { 5868 bkn_rx_add_tokens(sinfo, chan); 5869 } 5870 } 5871 5872 spin_unlock_irqrestore(&sinfo->lock, flags); 5873 5874 add_timer(&sinfo->rxtick); 5875 } 5876 5877 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4,15,0)) 5878 static void 5879 bkn_rxtick(unsigned long context) 5880 { 5881 bkn_switch_info_t *sinfo = (bkn_switch_info_t *)context; 5882 return bkn_rxtick_func(sinfo); 5883 } 5884 #else 5885 static void 5886 bkn_rxtick(struct timer_list *t) 5887 { 5888 bkn_switch_info_t *sinfo = from_timer(sinfo, t, rxtick); 5889 return bkn_rxtick_func(sinfo); 5890 } 5891 #endif 5892 5893 static void 5894 bkn_rx_rate_config(bkn_switch_info_t *sinfo) 5895 { 5896 unsigned long flags; 5897 int chan; 5898 uint32_t rxticks_per_sec, rps; 5899 uint32_t jiffies_per_rxtick; 5900 uint32_t tokens_per_rxtick; 5901 5902 spin_lock_irqsave(&sinfo->lock, flags); 5903 5904 /* Calculate the minimum update frequency across all channels */ 5905 rxticks_per_sec = 1; 5906 for (chan = 0; chan < NUM_RX_CHAN; chan++) { 5907 if (sinfo->rx[chan].burst_max == 0) { 5908 sinfo->rx[chan].burst_max = sinfo->rx[chan].rate_max / 10; 5909 } 5910 rps = sinfo->rx[chan].rate_max / sinfo->rx[chan].burst_max; 5911 if (rxticks_per_sec < rps) { 5912 rxticks_per_sec = rps; 5913 } 5914 } 5915 5916 /* Convert update frequency to system ticks */ 5917 jiffies_per_rxtick = HZ / rxticks_per_sec; 5918 if (jiffies_per_rxtick == 0) { 5919 jiffies_per_rxtick = 1; 5920 } 5921 rxticks_per_sec = HZ / jiffies_per_rxtick; 5922 5923 for (chan = 0; chan < NUM_RX_CHAN; chan++) { 5924 /* Ensure that burst size satifies overall rate */ 5925 tokens_per_rxtick = sinfo->rx[chan].rate_max / rxticks_per_sec; 5926 if (sinfo->rx[chan].burst_max < tokens_per_rxtick) { 5927 sinfo->rx[chan].burst_max = tokens_per_rxtick; 5928 } 5929 /* Ensure that rate has a sane value */ 5930 if (sinfo->rx[chan].rate_max != 0) { 5931 if (sinfo->rx[chan].rate_max < rxticks_per_sec) { 5932 sinfo->rx[chan].rate_max = rxticks_per_sec; 5933 } 5934 } 5935 sinfo->rx[chan].tokens = sinfo->rx[chan].burst_max; 5936 } 5937 5938 /* Update timer controls */ 5939 sinfo->rxticks_per_sec = rxticks_per_sec; 5940 sinfo->rxtick_jiffies = jiffies_per_rxtick; 5941 5942 spin_unlock_irqrestore(&sinfo->lock, flags); 5943 } 5944 5945 static void 5946 bkn_destroy_sinfo(bkn_switch_info_t *sinfo) 5947 { 5948 list_del(&sinfo->list); 5949 bkn_free_dcbs(sinfo); 5950 kfree(sinfo); 5951 } 5952 5953 static bkn_switch_info_t * 5954 bkn_create_sinfo(int dev_no) 5955 { 5956 bkn_switch_info_t *sinfo; 5957 int chan; 5958 5959 if ((sinfo = kmalloc(sizeof(*sinfo), GFP_KERNEL)) == NULL) { 5960 return NULL; 5961 } 5962 memset(sinfo, 0, sizeof(*sinfo)); 5963 INIT_LIST_HEAD(&sinfo->ndev_list); 5964 INIT_LIST_HEAD(&sinfo->rxpf_list); 5965 sinfo->base_addr = lkbde_get_dev_virt(dev_no); 5966 sinfo->dma_dev = lkbde_get_dma_dev(dev_no); 5967 sinfo->pdev = lkbde_get_hw_dev(dev_no); 5968 sinfo->dev_no = dev_no; 5969 sinfo->evt_idx = -1; 5970 5971 spin_lock_init(&sinfo->lock); 5972 skb_queue_head_init(&sinfo->tx_ptp_queue); 5973 INIT_WORK(&sinfo->tx_ptp_work, bkn_hw_tstamp_tx_work); 5974 5975 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4,15,0)) 5976 init_timer(&sinfo->timer); 5977 sinfo->timer.data = (unsigned long)sinfo; 5978 sinfo->timer.function = bkn_timer; 5979 #else 5980 timer_setup(&sinfo->timer, bkn_timer, 0); 5981 #endif 5982 sinfo->timer.expires = jiffies + 1; 5983 5984 INIT_LIST_HEAD(&sinfo->tx.api_dcb_list); 5985 5986 for (chan = 0; chan < NUM_RX_CHAN; chan++) { 5987 INIT_LIST_HEAD(&sinfo->rx[chan].api_dcb_list); 5988 sinfo->rx[chan].use_rx_skb = use_rx_skb; 5989 } 5990 5991 /* 5992 * Check for dual DMA mode where Rx DMA channel 0 uses DMA buffers 5993 * provided by the BCM API, and the remaining Rx DMA channel(s) 5994 * use socket buffers (SKB) provided by the Linux kernel. 5995 */ 5996 if (use_rx_skb == 2) { 5997 sinfo->rx[0].use_rx_skb = 0; 5998 } 5999 6000 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4,15,0)) 6001 init_timer(&sinfo->rxtick); 6002 sinfo->rxtick.data = (unsigned long)sinfo; 6003 sinfo->rxtick.function = bkn_rxtick; 6004 #else 6005 timer_setup(&sinfo->rxtick, bkn_rxtick, 0); 6006 #endif 6007 sinfo->rxtick.expires = jiffies + 1; 6008 6009 for (chan = 0; chan < NUM_RX_CHAN; chan++) { 6010 sinfo->rx[chan].rate_max = rx_rate[chan]; 6011 sinfo->rx[chan].burst_max = rx_burst[chan]; 6012 } 6013 bkn_rx_rate_config(sinfo); 6014 6015 add_timer(&sinfo->rxtick); 6016 6017 list_add_tail(&sinfo->list, &_sinfo_list); 6018 6019 return sinfo; 6020 } 6021 6022 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29)) 6023 static const struct net_device_ops bkn_netdev_ops = { 6024 .ndo_open = bkn_open, 6025 .ndo_stop = bkn_stop, 6026 .ndo_start_xmit = bkn_tx, 6027 .ndo_get_stats = bkn_get_stats, 6028 .ndo_validate_addr = eth_validate_addr, 6029 .ndo_set_rx_mode = bkn_set_multicast_list, 6030 .ndo_set_mac_address = bkn_set_mac_address, 6031 .ndo_do_ioctl = bkn_ioctl, 6032 .ndo_tx_timeout = NULL, 6033 .ndo_change_mtu = bkn_change_mtu, 6034 #ifdef CONFIG_NET_POLL_CONTROLLER 6035 .ndo_poll_controller = bkn_poll_controller, 6036 #endif 6037 }; 6038 #endif 6039 6040 static void 6041 bkn_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) 6042 { 6043 strlcpy(drvinfo->driver, "bcm-knet", sizeof(drvinfo->driver)); 6044 snprintf(drvinfo->version, sizeof(drvinfo->version), "%d", KCOM_VERSION); 6045 strlcpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version)); 6046 strlcpy(drvinfo->bus_info, "N/A", sizeof(drvinfo->bus_info)); 6047 } 6048 6049 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0)) 6050 static int 6051 bkn_get_ts_info(struct net_device *dev, struct ethtool_ts_info *info) 6052 { 6053 bkn_priv_t *priv = netdev_priv(dev); 6054 bkn_switch_info_t *sinfo = priv->sinfo; 6055 6056 switch (sinfo->dcb_type) { 6057 case 26: 6058 case 32: 6059 case 33: 6060 case 36: 6061 case 38: 6062 info->so_timestamping = SOF_TIMESTAMPING_TX_HARDWARE | 6063 SOF_TIMESTAMPING_TX_SOFTWARE | 6064 SOF_TIMESTAMPING_RX_HARDWARE | 6065 SOF_TIMESTAMPING_RX_SOFTWARE | 6066 SOF_TIMESTAMPING_SOFTWARE | 6067 SOF_TIMESTAMPING_RAW_HARDWARE; 6068 info->tx_types = 1 << HWTSTAMP_TX_OFF | 1 << HWTSTAMP_TX_ON; 6069 info->rx_filters = 1 << HWTSTAMP_FILTER_NONE | 1 << HWTSTAMP_FILTER_ALL; 6070 if (knet_hw_tstamp_ptp_clock_index_cb) { 6071 info->phc_index = knet_hw_tstamp_ptp_clock_index_cb(sinfo->dev_no); 6072 } else { 6073 info->phc_index = -1; 6074 } 6075 break; 6076 default: 6077 info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | 6078 SOF_TIMESTAMPING_RX_SOFTWARE | 6079 SOF_TIMESTAMPING_SOFTWARE; 6080 info->phc_index = -1; 6081 break; 6082 } 6083 6084 return 0; 6085 } 6086 #endif 6087 6088 static const struct ethtool_ops bkn_ethtool_ops = { 6089 .get_drvinfo = bkn_get_drvinfo, 6090 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0)) 6091 .get_ts_info = bkn_get_ts_info, 6092 #endif 6093 }; 6094 6095 static struct net_device * 6096 bkn_init_ndev(u8 *mac, char *name) 6097 { 6098 struct net_device *dev; 6099 6100 /* Create Ethernet device */ 6101 dev = alloc_etherdev(sizeof(bkn_priv_t)); 6102 6103 if (dev == NULL) { 6104 DBG_WARN(("Error allocating Ethernet device.\n")); 6105 return NULL; 6106 } 6107 #ifdef SET_MODULE_OWNER 6108 SET_MODULE_OWNER(dev); 6109 #endif 6110 6111 /* Set the device MAC address */ 6112 memcpy(dev->dev_addr, mac, 6); 6113 6114 /* Device information -- not available right now */ 6115 dev->irq = 0; 6116 dev->base_addr = 0; 6117 6118 /* Default MTU should not exceed MTU of switch front-panel ports */ 6119 dev->mtu = default_mtu; 6120 if (dev->mtu == 0) { 6121 dev->mtu = rx_buffer_size; 6122 } 6123 6124 /* Device vectors */ 6125 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29)) 6126 dev->netdev_ops = &bkn_netdev_ops; 6127 #else 6128 dev->open = bkn_open; 6129 dev->hard_start_xmit = bkn_tx; 6130 dev->stop = bkn_stop; 6131 dev->set_multicast_list = bkn_set_multicast_list; 6132 dev->do_ioctl = NULL; 6133 dev->get_stats = bkn_get_stats; 6134 dev->change_mtu = bkn_change_mtu; 6135 #ifdef CONFIG_NET_POLL_CONTROLLER 6136 dev->poll_controller = bkn_poll_controller; 6137 #endif 6138 #endif 6139 dev->ethtool_ops = &bkn_ethtool_ops; 6140 if (name && *name) { 6141 strncpy(dev->name, name, IFNAMSIZ-1); 6142 } 6143 6144 /* Register the kernel Ethernet device */ 6145 if (register_netdev(dev)) { 6146 DBG_WARN(("Error registering Ethernet device.\n")); 6147 free_netdev(dev); 6148 return NULL; 6149 } 6150 DBG_VERB(("Created Ethernet device %s.\n", dev->name)); 6151 6152 return dev; 6153 } 6154 6155 /* 6156 * Device Link Control Proc Read Entry 6157 */ 6158 static int 6159 bkn_proc_link_show(struct seq_file *m, void *v) 6160 { 6161 struct list_head *slist, *dlist; 6162 struct net_device *dev; 6163 bkn_priv_t *priv; 6164 bkn_switch_info_t *sinfo; 6165 unsigned long flags; 6166 6167 seq_printf(m, "Software link status:\n"); 6168 list_for_each(slist, &_sinfo_list) { 6169 sinfo = (bkn_switch_info_t *)slist; 6170 spin_lock_irqsave(&sinfo->lock, flags); 6171 list_for_each(dlist, &sinfo->ndev_list) { 6172 priv = (bkn_priv_t *)dlist; 6173 dev = priv->dev; 6174 if (dev) { 6175 seq_printf(m, " %-14s %s\n", dev->name, 6176 netif_carrier_ok(dev) ? "up" : "down"); 6177 } 6178 } 6179 spin_unlock_irqrestore(&sinfo->lock, flags); 6180 } 6181 return 0; 6182 } 6183 6184 static int 6185 bkn_proc_link_open(struct inode * inode, struct file * file) 6186 { 6187 return single_open(file, bkn_proc_link_show, NULL); 6188 } 6189 6190 /* 6191 * Device Link Control Proc Write Entry 6192 * 6193 * Syntax: 6194 * <netif>=up|down 6195 * 6196 * Where <netif> is a virtual network interface name. 6197 * 6198 * Examples: 6199 * eth4=up 6200 * eth4=down 6201 */ 6202 static ssize_t 6203 bkn_proc_link_write(struct file *file, const char *buf, 6204 size_t count, loff_t *loff) 6205 { 6206 struct list_head *slist, *dlist; 6207 struct net_device *dev; 6208 bkn_priv_t *priv; 6209 bkn_switch_info_t *sinfo; 6210 unsigned long flags; 6211 char link_str[40]; 6212 char *ptr; 6213 char *newline; 6214 6215 if (count > sizeof(link_str)) { 6216 count = sizeof(link_str) - 1; 6217 link_str[count] = '\0'; 6218 } 6219 if (copy_from_user(link_str, buf, count)) { 6220 return -EFAULT; 6221 } 6222 link_str[count] = 0; 6223 newline = strchr(link_str, '\n'); 6224 if (newline) { 6225 /* Chop off the trailing newline */ 6226 *newline = '\0'; 6227 } 6228 6229 if ((ptr = strchr(link_str, '=')) == NULL && 6230 (ptr = strchr(link_str, ':')) == NULL) { 6231 gprintk("Error: link syntax not recognized: '%s'\n", link_str); 6232 return count; 6233 } 6234 *ptr++ = 0; 6235 6236 dev = NULL; 6237 list_for_each(slist, &_sinfo_list) { 6238 sinfo = (bkn_switch_info_t *)slist; 6239 spin_lock_irqsave(&sinfo->lock, flags); 6240 list_for_each(dlist, &sinfo->ndev_list) { 6241 priv = (bkn_priv_t *)dlist; 6242 if (priv->dev) { 6243 if (strcmp(priv->dev->name, link_str) == 0) { 6244 dev = priv->dev; 6245 break; 6246 } 6247 } 6248 } 6249 if (dev) { 6250 if (strcmp(ptr, "up") == 0) { 6251 netif_carrier_on(dev); 6252 } else if (strcmp(ptr, "down") == 0) { 6253 netif_carrier_off(dev); 6254 } else { 6255 gprintk("Warning: unknown link state setting: '%s'\n", ptr); 6256 } 6257 spin_unlock_irqrestore(&sinfo->lock, flags); 6258 return count; 6259 } 6260 spin_unlock_irqrestore(&sinfo->lock, flags); 6261 } 6262 6263 gprintk("Warning: unknown network interface: '%s'\n", link_str); 6264 6265 return count; 6266 } 6267 6268 struct file_operations bkn_proc_link_file_ops = { 6269 owner: THIS_MODULE, 6270 open: bkn_proc_link_open, 6271 read: seq_read, 6272 llseek: seq_lseek, 6273 write: bkn_proc_link_write, 6274 release: single_release, 6275 }; 6276 6277 /* 6278 * Device Rate Control Proc Read Entry 6279 */ 6280 static int 6281 bkn_proc_rate_show(struct seq_file *m, void *v) 6282 { 6283 int unit = 0; 6284 struct list_head *list; 6285 bkn_switch_info_t *sinfo; 6286 int chan; 6287 6288 list_for_each(list, &_sinfo_list) { 6289 sinfo = (bkn_switch_info_t *)list; 6290 6291 seq_printf(m, "Rate control (unit %d):\n", unit); 6292 for (chan = 0; chan < sinfo->rx_chans; chan++) { 6293 seq_printf(m, " Rx%d max rate %8u\n", 6294 chan, sinfo->rx[chan].rate_max); 6295 seq_printf(m, " Rx%d max burst %8u\n", 6296 chan, sinfo->rx[chan].burst_max); 6297 seq_printf(m, " Rx%d rate %8u\n", 6298 chan, sinfo->rx[chan].rate); 6299 seq_printf(m, " Rx%d tokens %8u\n", 6300 chan, sinfo->rx[chan].tokens); 6301 } 6302 6303 unit++; 6304 } 6305 return 0; 6306 } 6307 6308 static int 6309 bkn_proc_rate_open(struct inode * inode, struct file * file) 6310 { 6311 return single_open(file, bkn_proc_rate_show, NULL); 6312 } 6313 6314 /* 6315 * Device Rate Control Proc Write Entry 6316 * 6317 * Syntax: 6318 * [<unit>:]rx_rate=<rate0>[,<rate1>[,<rate2]] 6319 * 6320 * Where <rate0> is packets/sec for the first Rx DMA channel, 6321 * <rate1> is packets/sec for the second Rx DMA channel, etc. 6322 * 6323 * Examples: 6324 * rx_rate=5000 6325 * 0:rx_rate=10000,10000 6326 * 1:rx_rate=10000,5000 6327 */ 6328 static ssize_t 6329 bkn_proc_rate_write(struct file *file, const char *buf, 6330 size_t count, loff_t *loff) 6331 { 6332 bkn_switch_info_t *sinfo; 6333 char rate_str[80]; 6334 char *ptr; 6335 int unit, chan; 6336 6337 if (count > sizeof(rate_str)) { 6338 count = sizeof(rate_str) - 1; 6339 rate_str[count] = '\0'; 6340 } 6341 if (copy_from_user(rate_str, buf, count)) { 6342 return -EFAULT; 6343 } 6344 6345 unit = simple_strtol(rate_str, NULL, 10); 6346 sinfo = bkn_sinfo_from_unit(unit); 6347 if (sinfo == NULL) { 6348 gprintk("Warning: unknown unit: %d\n", unit); 6349 return count; 6350 } 6351 6352 if ((ptr = strstr(rate_str, "rx_rate=")) != NULL) { 6353 ptr += 7; 6354 chan = 0; 6355 do { 6356 ptr++; 6357 sinfo->rx[chan].rate_max = simple_strtol(ptr, NULL, 10); 6358 } while ((ptr = strchr(ptr, ',')) != NULL && ++chan < sinfo->rx_chans); 6359 bkn_rx_rate_config(sinfo); 6360 } else if ((ptr = strstr(rate_str, "rx_burst=")) != NULL) { 6361 ptr += 8; 6362 chan = 0; 6363 do { 6364 ptr++; 6365 sinfo->rx[chan].burst_max = simple_strtol(ptr, NULL, 10); 6366 } while ((ptr = strchr(ptr, ',')) != NULL && ++chan < sinfo->rx_chans); 6367 bkn_rx_rate_config(sinfo); 6368 } else { 6369 gprintk("Warning: unknown configuration setting\n"); 6370 } 6371 6372 return count; 6373 } 6374 6375 struct file_operations bkn_proc_rate_file_ops = { 6376 owner: THIS_MODULE, 6377 open: bkn_proc_rate_open, 6378 read: seq_read, 6379 llseek: seq_lseek, 6380 write: bkn_proc_rate_write, 6381 release: single_release, 6382 }; 6383 6384 /* 6385 * Driver DMA Proc Entry 6386 * 6387 * This output can be rather large (> PAGE_SIZE) so we use the 6388 * seq_file interface to do the output. Special header records 6389 * are indicated by a negative DCB index. 6390 */ 6391 typedef struct { 6392 int dev_no; /* Current unit */ 6393 int rx_dma; /* 0: Tx DMA, 1: Rx DMA*/ 6394 int ch_no; /* DMA channel no. (Rx only) */ 6395 int idx; /* DCB index */ 6396 } bkn_seq_dma_iter_t; 6397 6398 /* From current record, move forward 'pos' records */ 6399 static int 6400 bkn_seq_dma_next_pos(bkn_seq_dma_iter_t *iter, loff_t pos) 6401 { 6402 bkn_switch_info_t *sinfo; 6403 6404 sinfo = bkn_sinfo_from_unit(iter->dev_no); 6405 while (pos) { 6406 if (iter->rx_dma) { 6407 if (++iter->idx >= MAX_RX_DCBS + 1) { 6408 iter->idx = -1; 6409 if (++iter->ch_no >= sinfo->rx_chans) { 6410 iter->rx_dma = 0; 6411 iter->ch_no = 0; 6412 iter->dev_no++; 6413 if ((sinfo = bkn_sinfo_from_unit(iter->dev_no)) == NULL) { 6414 return -1; 6415 } 6416 } 6417 } 6418 } else { 6419 if (++iter->idx >= MAX_TX_DCBS + 1) { 6420 iter->idx = -1; 6421 iter->rx_dma = 1; 6422 } 6423 } 6424 pos--; 6425 } 6426 return 0; 6427 } 6428 6429 /* Initialize private data and move to requested start record */ 6430 static void * 6431 bkn_seq_dma_start(struct seq_file *s, loff_t *pos) 6432 { 6433 bkn_seq_dma_iter_t *iter; 6434 6435 iter = kmalloc(sizeof(bkn_seq_dma_iter_t), GFP_KERNEL); 6436 if (!iter) { 6437 return NULL; 6438 } 6439 memset(iter, 0, sizeof(*iter)); 6440 iter->idx = -2; 6441 if (bkn_seq_dma_next_pos(iter, *pos) < 0) { 6442 kfree(iter); 6443 return NULL; 6444 } 6445 return iter; 6446 } 6447 6448 /* Move to next record */ 6449 static void * 6450 bkn_seq_dma_next(struct seq_file *s, void *v, loff_t *pos) 6451 { 6452 bkn_seq_dma_iter_t *iter = (bkn_seq_dma_iter_t *)v; 6453 void *rv = iter; 6454 6455 if (bkn_seq_dma_next_pos(iter, 1) < 0) { 6456 return NULL; 6457 } 6458 (*pos)++; 6459 return rv; 6460 } 6461 6462 /* Release private data */ 6463 static void 6464 bkn_seq_dma_stop(struct seq_file *s, void *v) 6465 { 6466 if (v) { 6467 kfree(v); 6468 } 6469 } 6470 6471 /* Print current record */ 6472 static int 6473 bkn_seq_dma_show(struct seq_file *s, void *v) 6474 { 6475 bkn_seq_dma_iter_t *iter = (bkn_seq_dma_iter_t *)v; 6476 bkn_switch_info_t *sinfo; 6477 uint32_t *dcb = NULL; 6478 bkn_dcb_chain_t *dcb_chain = NULL; 6479 struct list_head *curr, *next; 6480 unsigned long flags; 6481 int chan, cnt; 6482 6483 sinfo = bkn_sinfo_from_unit(iter->dev_no); 6484 if (sinfo == NULL) { 6485 /* Should not happen */ 6486 return 0; 6487 } 6488 6489 if (iter->rx_dma == 0) { 6490 if (iter->idx == -2) { 6491 seq_printf(s, "Pending events: 0x%x\n", sinfo->dma_events); 6492 } else if (iter->idx == -1) { 6493 spin_lock_irqsave(&sinfo->lock, flags); 6494 curr = &sinfo->tx.api_dcb_list; 6495 dcb_chain = sinfo->tx.api_dcb_chain; 6496 while (dcb_chain) { 6497 seq_printf(s, " [0x%08lx]--->\n", (unsigned long)dcb_chain->dcb_dma); 6498 for (cnt = 0; cnt < dcb_chain->dcb_cnt; cnt++) { 6499 dcb = &dcb_chain->dcb_mem[sinfo->dcb_wsize * cnt]; 6500 if (sinfo->cmic_type == 'x') { 6501 seq_printf(s, " DCB %2d: 0x%08x 0x%08x 0x%08x 0x%08x\n", cnt, 6502 dcb[0], dcb[1], dcb[2], dcb[sinfo->dcb_wsize-1]); 6503 } else { 6504 seq_printf(s, " DCB %2d: 0x%08x 0x%08x ... 0x%08x\n", cnt, 6505 dcb[0], dcb[1], dcb[sinfo->dcb_wsize-1]); 6506 } 6507 } 6508 next = curr->next; 6509 if (next != &sinfo->tx.api_dcb_list) { 6510 dcb_chain = list_entry(next, bkn_dcb_chain_t, list); 6511 curr = next; 6512 } else { 6513 dcb_chain = NULL; 6514 } 6515 } 6516 dcb = NULL; 6517 spin_unlock_irqrestore(&sinfo->lock, flags); 6518 seq_printf(s, 6519 "Tx DCB info (unit %d):\n" 6520 " api: %d\n" 6521 " dirty: %d\n" 6522 " cur: %d\n" 6523 " free: %d\n" 6524 " pause: %s\n", 6525 iter->dev_no, 6526 sinfo->tx.api_active, 6527 sinfo->tx.dirty, 6528 sinfo->tx.cur, 6529 sinfo->tx.free, 6530 netif_queue_stopped(sinfo->dev) ? "yes" : "no"); 6531 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN)) { 6532 bkn_desc_info_t *desc = &sinfo->tx.desc[0]; 6533 uint64_t halt = sinfo->halt_addr[XGS_DMA_TX_CHAN]; 6534 seq_printf(s, 6535 " halt: %d\n", 6536 (int)((uint32_t)(halt - desc->dcb_dma) / (sinfo->dcb_wsize * sizeof(uint32_t)))); 6537 } 6538 } else { 6539 dcb = sinfo->tx.desc[iter->idx].dcb_mem; 6540 if (iter->idx == 0) { 6541 seq_printf(s, " [0x%08lx]--->\n", (unsigned long)sinfo->tx.desc[0].dcb_dma); 6542 } 6543 } 6544 } else { 6545 if (iter->idx == -1) { 6546 chan = iter->ch_no; 6547 seq_printf(s, 6548 "Rx%d DCB info (unit %d):\n" 6549 " api: %d\n" 6550 " dirty: %d\n" 6551 " cur: %d\n" 6552 " free: %d\n" 6553 " run: %d\n", 6554 chan, iter->dev_no, 6555 sinfo->rx[chan].api_active, 6556 sinfo->rx[chan].dirty, 6557 sinfo->rx[chan].cur, 6558 sinfo->rx[chan].free, 6559 sinfo->rx[chan].running); 6560 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan) && sinfo->rx[chan].use_rx_skb) { 6561 bkn_desc_info_t *desc = &sinfo->rx[chan].desc[0]; 6562 uint64_t halt = sinfo->halt_addr[XGS_DMA_RX_CHAN + chan]; 6563 seq_printf(s, 6564 " halt: %d\n", 6565 (int)((uint32_t)(halt - desc->dcb_dma) / (sinfo->dcb_wsize * sizeof(uint32_t)))); 6566 } 6567 } else if (sinfo->rx[iter->ch_no].use_rx_skb) { 6568 dcb = sinfo->rx[iter->ch_no].desc[iter->idx].dcb_mem; 6569 if (iter->idx == 0) { 6570 seq_printf(s, " [0x%08lx]--->\n", (unsigned long)sinfo->rx[iter->ch_no].desc[0].dcb_dma); 6571 } 6572 } else if (!sinfo->rx[iter->ch_no].use_rx_skb && iter->idx == 0) { 6573 spin_lock_irqsave(&sinfo->lock, flags); 6574 curr = &sinfo->rx[iter->ch_no].api_dcb_list; 6575 dcb_chain = sinfo->rx[iter->ch_no].api_dcb_chain; 6576 while (dcb_chain) { 6577 seq_printf(s, " [0x%08lx]--->\n", (unsigned long)dcb_chain->dcb_dma); 6578 for (cnt = 0; cnt < dcb_chain->dcb_cnt; cnt++) { 6579 dcb = &dcb_chain->dcb_mem[sinfo->dcb_wsize * cnt]; 6580 if (sinfo->cmic_type == 'x') { 6581 seq_printf(s, " DCB %2d: 0x%08x 0x%08x 0x%08x 0x%08x\n", cnt, 6582 dcb[0], dcb[1], dcb[2], dcb[sinfo->dcb_wsize-1]); 6583 } else { 6584 seq_printf(s, " DCB %2d: 0x%08x 0x%08x ... 0x%08x\n", cnt, 6585 dcb[0], dcb[1], dcb[sinfo->dcb_wsize-1]); 6586 } 6587 } 6588 next = curr->next; 6589 if (next != &sinfo->rx[iter->ch_no].api_dcb_list) { 6590 dcb_chain = list_entry(next, bkn_dcb_chain_t, list); 6591 curr = next; 6592 } else { 6593 dcb_chain = NULL; 6594 } 6595 } 6596 dcb = NULL; 6597 spin_unlock_irqrestore(&sinfo->lock, flags); 6598 } 6599 } 6600 if (dcb) { 6601 if (sinfo->cmic_type == 'x') { 6602 seq_printf(s, " DCB %2d: 0x%08x 0x%08x 0x%08x 0x%08x\n", iter->idx, 6603 dcb[0], dcb[1], dcb[2], dcb[sinfo->dcb_wsize-1]); 6604 } else { 6605 seq_printf(s, " DCB %2d: 0x%08x 0x%08x ... 0x%08x\n", iter->idx, 6606 dcb[0], dcb[1], dcb[sinfo->dcb_wsize-1]); 6607 } 6608 } 6609 return 0; 6610 } 6611 6612 static struct seq_operations bkn_seq_dma_ops = { 6613 .start = bkn_seq_dma_start, 6614 .next = bkn_seq_dma_next, 6615 .stop = bkn_seq_dma_stop, 6616 .show = bkn_seq_dma_show 6617 }; 6618 6619 static int 6620 bkn_seq_dma_open(struct inode *inode, struct file *file) 6621 { 6622 return seq_open(file, &bkn_seq_dma_ops); 6623 }; 6624 6625 static struct file_operations bkn_seq_dma_file_ops = { 6626 .owner = THIS_MODULE, 6627 .open = bkn_seq_dma_open, 6628 .read = seq_read, 6629 .llseek = seq_lseek, 6630 .release = seq_release 6631 }; 6632 6633 /* 6634 * Device Debug Control Proc Write Entry 6635 * 6636 * Syntax: 6637 * [<unit>:]debug=<mask> 6638 * 6639 * Where <mask> corresponds to the debug module parameter. 6640 * 6641 * Examples: 6642 * debug=0xffff 6643 * 0:debug-0x2000 6644 */ 6645 static ssize_t 6646 bkn_proc_debug_write(struct file *file, const char *buf, 6647 size_t count, loff_t *loff) 6648 { 6649 bkn_switch_info_t *sinfo; 6650 char debug_str[40]; 6651 char *ptr; 6652 int unit; 6653 6654 if (count > sizeof(debug_str)) { 6655 count = sizeof(debug_str) - 1; 6656 debug_str[count] = '\0'; 6657 } 6658 if (copy_from_user(debug_str, buf, count)) { 6659 return -EFAULT; 6660 } 6661 6662 unit = simple_strtol(debug_str, NULL, 10); 6663 sinfo = bkn_sinfo_from_unit(unit); 6664 if (sinfo == NULL) { 6665 gprintk("Warning: unknown unit: %d\n", unit); 6666 return count; 6667 } 6668 6669 if ((ptr = strstr(debug_str, "debug=")) != NULL) { 6670 ptr += 6; 6671 debug = simple_strtol(ptr, NULL, 0); 6672 } else { 6673 gprintk("Warning: unknown configuration setting\n"); 6674 } 6675 6676 return count; 6677 } 6678 6679 /* 6680 * Driver Debug Proc Entry 6681 */ 6682 static int 6683 bkn_proc_debug_show(struct seq_file *m, void *v) 6684 { 6685 int unit = 0; 6686 struct list_head *list; 6687 bkn_switch_info_t *sinfo; 6688 6689 seq_printf(m, "Configuration:\n"); 6690 seq_printf(m, " debug: 0x%x\n", debug); 6691 seq_printf(m, " mac_addr: %02x:%02x:%02x:%02x:%02x:%02x\n", 6692 bkn_dev_mac[0], bkn_dev_mac[1], bkn_dev_mac[2], 6693 bkn_dev_mac[3], bkn_dev_mac[4], bkn_dev_mac[5]); 6694 seq_printf(m, " rx_buffer_size: %d (0x%x)\n", 6695 rx_buffer_size, rx_buffer_size); 6696 seq_printf(m, " rcpu_mode: %d\n", rcpu_mode); 6697 seq_printf(m, " rcpu_dmac: %02x:%02x:%02x:%02x:%02x:%02x\n", 6698 bkn_rcpu_dmac[0], bkn_rcpu_dmac[1], bkn_rcpu_dmac[2], 6699 bkn_rcpu_dmac[3], bkn_rcpu_dmac[4], bkn_rcpu_dmac[5]); 6700 seq_printf(m, " rcpu_smac: %02x:%02x:%02x:%02x:%02x:%02x\n", 6701 bkn_rcpu_smac[0], bkn_rcpu_smac[1], bkn_rcpu_smac[2], 6702 bkn_rcpu_smac[3], bkn_rcpu_smac[4], bkn_rcpu_smac[5]); 6703 seq_printf(m, " rcpu_ethertype: 0x%x\n", rcpu_ethertype); 6704 seq_printf(m, " rcpu_signature: 0x%x\n", rcpu_signature); 6705 seq_printf(m, " rcpu_vlan: %d\n", rcpu_vlan); 6706 seq_printf(m, " use_rx_skb: %d\n", use_rx_skb); 6707 seq_printf(m, " num_rx_prio: %d\n", num_rx_prio); 6708 seq_printf(m, " check_rcpu_sig: %d\n", check_rcpu_signature); 6709 seq_printf(m, " default_mtu: %d\n", default_mtu); 6710 seq_printf(m, " rx_sync_retry: %d\n", rx_sync_retry); 6711 seq_printf(m, " use_napi: %d\n", use_napi); 6712 seq_printf(m, " napi_weight: %d\n", napi_weight); 6713 seq_printf(m, " basedev_susp: %d\n", basedev_suspend); 6714 seq_printf(m, "Thread states:\n"); 6715 seq_printf(m, " Command thread: %d\n", bkn_cmd_ctrl.state); 6716 seq_printf(m, " Event thread: %d\n", bkn_evt_ctrl.state); 6717 seq_printf(m, "Active IOCTLs:\n"); 6718 seq_printf(m, " Command: %d\n", ioctl_cmd); 6719 seq_printf(m, " Event: %d\n", ioctl_evt); 6720 6721 list_for_each(list, &_sinfo_list) { 6722 sinfo = (bkn_switch_info_t *)list; 6723 6724 seq_printf(m, "Device %d:\n", unit); 6725 seq_printf(m, " base_addr: 0x%p\n", sinfo->base_addr); 6726 seq_printf(m, " dev_no: %d\n", sinfo->dev_no); 6727 seq_printf(m, " cmic_type: %c\n", sinfo->cmic_type); 6728 seq_printf(m, " dcb_type: %d\n", sinfo->dcb_type); 6729 seq_printf(m, " dcb_wsize: %d\n", sinfo->dcb_wsize); 6730 seq_printf(m, " pkt_hdr_size: %d\n", sinfo->pkt_hdr_size); 6731 seq_printf(m, " rx_chans: %d\n", sinfo->rx_chans); 6732 seq_printf(m, " cdma_chans: 0x%x\n", sinfo->cdma_channels); 6733 seq_printf(m, " irq_mask: 0x%x\n", sinfo->irq_mask); 6734 seq_printf(m, " dma_events: 0x%x\n", sinfo->dma_events); 6735 seq_printf(m, " dcb_dma: 0x%p\n", (void *)(sal_paddr_t)sinfo->dcb_dma); 6736 seq_printf(m, " dcb_mem_size: 0x%x\n", sinfo->dcb_mem_size); 6737 seq_printf(m, " rcpu_sig: 0x%x\n", sinfo->rcpu_sig); 6738 seq_printf(m, " napi_poll_mode: %d\n", sinfo->napi_poll_mode); 6739 seq_printf(m, " inst_id: 0x%x\n", sinfo->inst_id); 6740 seq_printf(m, " evt_queue: %d\n", sinfo->evt_idx); 6741 6742 unit++; 6743 } 6744 6745 return 0; 6746 } 6747 6748 static int bkn_proc_debug_open(struct inode * inode, struct file * file) 6749 { 6750 return single_open(file, bkn_proc_debug_show, NULL); 6751 } 6752 6753 struct file_operations bkn_proc_debug_file_ops = { 6754 owner: THIS_MODULE, 6755 open: bkn_proc_debug_open, 6756 read: seq_read, 6757 llseek: seq_lseek, 6758 write: bkn_proc_debug_write, 6759 release: single_release, 6760 }; 6761 6762 /* 6763 * Device Statistics Proc Entry 6764 */ 6765 static int 6766 bkn_proc_stats_show(struct seq_file *m, void *v) 6767 { 6768 int unit = 0; 6769 struct list_head *list, *flist; 6770 bkn_switch_info_t *sinfo; 6771 bkn_filter_t *filter; 6772 int chan; 6773 6774 6775 list_for_each(list, &_sinfo_list) { 6776 sinfo = (bkn_switch_info_t *)list; 6777 6778 seq_printf(m, "Device stats (unit %d):\n", unit); 6779 seq_printf(m, " Interrupts %10u\n", sinfo->interrupts); 6780 seq_printf(m, " Tx packets %10u\n", sinfo->tx.pkts); 6781 for (chan = 0; chan < sinfo->rx_chans; chan++) { 6782 seq_printf(m, " Rx%d packets %10u\n", chan, sinfo->rx[chan].pkts); 6783 } 6784 for (chan = 0; chan < sinfo->rx_chans; chan++) { 6785 if (sinfo->interrupts == 0) { 6786 /* Avoid divide-by-zero */ 6787 seq_printf(m, " Rx%d pkts/intr -\n", chan); 6788 } else { 6789 seq_printf(m, " Rx%d pkts/intr %8u\n", 6790 chan, sinfo->rx[chan].pkts / sinfo->interrupts); 6791 } 6792 } 6793 seq_printf(m, " Timer runs %10u\n", sinfo->timer_runs); 6794 seq_printf(m, " NAPI reruns %10u\n", sinfo->napi_not_done); 6795 6796 list_for_each(flist, &sinfo->rxpf_list) { 6797 filter = (bkn_filter_t *)flist; 6798 6799 seq_printf(m, " Filter %d stats:\n", filter->kf.id); 6800 seq_printf(m, " Hits %10lu\n", filter->hits); 6801 } 6802 6803 unit++; 6804 } 6805 return 0; 6806 } 6807 6808 static int bkn_proc_stats_open(struct inode * inode, struct file * file) 6809 { 6810 return single_open(file, bkn_proc_stats_show, NULL); 6811 } 6812 6813 /* 6814 * Device Statistics Proc Write Entry 6815 * 6816 * Syntax: 6817 * [<unit>:]clear[=all] 6818 * 6819 * Where <mask> corresponds to the debug module parameter. 6820 * 6821 * Examples: 6822 * clear 6823 * 0:clear=all 6824 */ 6825 static ssize_t 6826 bkn_proc_stats_write(struct file *file, const char *buf, 6827 size_t count, loff_t *loff) 6828 { 6829 bkn_switch_info_t *sinfo; 6830 struct list_head *flist; 6831 bkn_filter_t *filter; 6832 char debug_str[40]; 6833 char *ptr; 6834 int unit; 6835 int clear_mask; 6836 int chan; 6837 6838 if (count > sizeof(debug_str)) { 6839 count = sizeof(debug_str) - 1; 6840 debug_str[count] = '\0'; 6841 } 6842 if (copy_from_user(debug_str, buf, count)) { 6843 return -EFAULT; 6844 } 6845 6846 unit = simple_strtol(debug_str, NULL, 10); 6847 sinfo = bkn_sinfo_from_unit(unit); 6848 if (sinfo == NULL) { 6849 gprintk("Warning: unknown unit: %d\n", unit); 6850 return count; 6851 } 6852 6853 clear_mask = 0; 6854 if ((ptr = strstr(debug_str, "clear=")) != NULL) { 6855 ptr += 6; 6856 if (strncmp(ptr, "all", 3) == 0) { 6857 clear_mask = ~0; 6858 } 6859 } else if ((ptr = strstr(debug_str, "clear")) != NULL) { 6860 clear_mask = ~0; 6861 } else { 6862 gprintk("Warning: unknown configuration setting\n"); 6863 } 6864 6865 if (clear_mask) { 6866 sinfo->tx.pkts = 0; 6867 for (chan = 0; chan < sinfo->rx_chans; chan++) { 6868 sinfo->rx[chan].pkts = 0; 6869 } 6870 sinfo->interrupts = 0; 6871 sinfo->timer_runs = 0; 6872 sinfo->napi_not_done = 0; 6873 list_for_each(flist, &sinfo->rxpf_list) { 6874 filter = (bkn_filter_t *)flist; 6875 filter->hits = 0; 6876 } 6877 } 6878 6879 return count; 6880 } 6881 6882 struct file_operations bkn_proc_stats_file_ops = { 6883 owner: THIS_MODULE, 6884 open: bkn_proc_stats_open, 6885 read: seq_read, 6886 llseek: seq_lseek, 6887 write: bkn_proc_stats_write, 6888 release: single_release, 6889 }; 6890 6891 /* 6892 * Device Debug Statistics Proc Entry 6893 */ 6894 static int 6895 bkn_proc_dstats_show(struct seq_file *m, void *v) 6896 { 6897 int unit = 0; 6898 struct list_head *list; 6899 bkn_switch_info_t *sinfo; 6900 int chan; 6901 6902 list_for_each(list, &_sinfo_list) { 6903 sinfo = (bkn_switch_info_t *)list; 6904 6905 seq_printf(m, "Device debug stats (unit %d):\n", unit); 6906 seq_printf(m, " Tx drop no skb %10u\n", 6907 sinfo->tx.pkts_d_no_skb); 6908 seq_printf(m, " Tx drop rcpu encap %10u\n", 6909 sinfo->tx.pkts_d_rcpu_encap); 6910 seq_printf(m, " Tx drop rcpu sig %10u\n", 6911 sinfo->tx.pkts_d_rcpu_sig); 6912 seq_printf(m, " Tx drop rcpu meta %10u\n", 6913 sinfo->tx.pkts_d_rcpu_meta); 6914 seq_printf(m, " Tx drop pad failed %10u\n", 6915 sinfo->tx.pkts_d_pad_fail); 6916 seq_printf(m, " Tx drop no resource %10u\n", 6917 sinfo->tx.pkts_d_dma_resrc); 6918 seq_printf(m, " Tx drop callback %10u\n", 6919 sinfo->tx.pkts_d_callback); 6920 seq_printf(m, " Tx drop no link %10u\n", 6921 sinfo->tx.pkts_d_no_link); 6922 seq_printf(m, " Tx drop oversized %10u\n", 6923 sinfo->tx.pkts_d_over_limit); 6924 seq_printf(m, " Tx suspends %10u\n", 6925 sinfo->tx.suspends); 6926 for (chan = 0; chan < sinfo->rx_chans; chan++) { 6927 seq_printf(m, " Rx%d filter to api %10u\n", 6928 chan, sinfo->rx[chan].pkts_f_api); 6929 seq_printf(m, " Rx%d filter to netif %10u\n", 6930 chan, sinfo->rx[chan].pkts_f_netif); 6931 seq_printf(m, " Rx%d mirror to api %10u\n", 6932 chan, sinfo->rx[chan].pkts_m_api); 6933 seq_printf(m, " Rx%d mirror to netif %10u\n", 6934 chan, sinfo->rx[chan].pkts_m_netif); 6935 seq_printf(m, " Rx%d drop no skb %10u\n", 6936 chan, sinfo->rx[chan].pkts_d_no_skb); 6937 seq_printf(m, " Rx%d drop no match %10u\n", 6938 chan, sinfo->rx[chan].pkts_d_no_match); 6939 seq_printf(m, " Rx%d drop unkn netif %10u\n", 6940 chan, sinfo->rx[chan].pkts_d_unkn_netif); 6941 seq_printf(m, " Rx%d drop unkn dest %10u\n", 6942 chan, sinfo->rx[chan].pkts_d_unkn_dest); 6943 seq_printf(m, " Rx%d drop callback %10u\n", 6944 chan, sinfo->rx[chan].pkts_d_callback); 6945 seq_printf(m, " Rx%d drop no link %10u\n", 6946 chan, sinfo->rx[chan].pkts_d_no_link); 6947 seq_printf(m, " Rx%d sync error %10u\n", 6948 chan, sinfo->rx[chan].sync_err); 6949 seq_printf(m, " Rx%d sync retry %10u\n", 6950 chan, sinfo->rx[chan].sync_retry); 6951 seq_printf(m, " Rx%d sync maxloop %10u\n", 6952 chan, sinfo->rx[chan].sync_maxloop); 6953 seq_printf(m, " Rx%d drop no buffer %10u\n", 6954 chan, sinfo->rx[chan].pkts_d_no_api_buf); 6955 } 6956 unit++; 6957 } 6958 return 0; 6959 } 6960 6961 static int bkn_proc_dstats_open(struct inode * inode, struct file * file) 6962 { 6963 return single_open(file, bkn_proc_dstats_show, NULL); 6964 } 6965 6966 /* 6967 * Device Debug Statistics Proc Write Entry 6968 * 6969 * Syntax: 6970 * [<unit>:]clear[=all|tx|rx[<chan>]] 6971 * 6972 * Where <mask> corresponds to the debug module parameter. 6973 * 6974 * Examples: 6975 * clear 6976 * 0:clear=rx1 6977 */ 6978 static ssize_t 6979 bkn_proc_dstats_write(struct file *file, const char *buf, 6980 size_t count, loff_t *loff) 6981 { 6982 bkn_switch_info_t *sinfo; 6983 char debug_str[40]; 6984 char *ptr; 6985 int unit; 6986 int clear_mask; 6987 int chan; 6988 6989 if (count > sizeof(debug_str)) { 6990 count = sizeof(debug_str) - 1; 6991 debug_str[count] = '\0'; 6992 } 6993 if (copy_from_user(debug_str, buf, count)) { 6994 return -EFAULT; 6995 } 6996 6997 unit = simple_strtol(debug_str, NULL, 10); 6998 sinfo = bkn_sinfo_from_unit(unit); 6999 if (sinfo == NULL) { 7000 gprintk("Warning: unknown unit: %d\n", unit); 7001 return count; 7002 } 7003 7004 clear_mask = 0; 7005 if ((ptr = strstr(debug_str, "clear=")) != NULL) { 7006 ptr += 6; 7007 if (strncmp(ptr, "all", 3) == 0) { 7008 clear_mask = ~0; 7009 } else if (strncmp(ptr, "dev", 3) == 0) { 7010 clear_mask = 0x20; 7011 } else if (strncmp(ptr, "tx", 2) == 0) { 7012 clear_mask = 0x10; 7013 } else if (strncmp(ptr, "rx0", 3) == 0) { 7014 clear_mask = (1 << 0); 7015 } else if (strncmp(ptr, "rx1", 3) == 0) { 7016 clear_mask = (1 << 1); 7017 } else if (strncmp(ptr, "rx", 2) == 0) { 7018 clear_mask = 0xf; 7019 } 7020 } else if ((ptr = strstr(debug_str, "clear")) != NULL) { 7021 clear_mask = ~0; 7022 } else { 7023 gprintk("Warning: unknown configuration setting\n"); 7024 } 7025 7026 /* Tx counters */ 7027 if (clear_mask & 0x10) { 7028 sinfo->tx.pkts_d_no_skb = 0; 7029 sinfo->tx.pkts_d_rcpu_encap = 0; 7030 sinfo->tx.pkts_d_rcpu_sig = 0; 7031 sinfo->tx.pkts_d_rcpu_meta = 0; 7032 sinfo->tx.pkts_d_pad_fail = 0; 7033 sinfo->tx.pkts_d_over_limit = 0; 7034 sinfo->tx.pkts_d_dma_resrc = 0; 7035 sinfo->tx.suspends = 0; 7036 } 7037 /* Rx counters */ 7038 for (chan = 0; chan < sinfo->rx_chans; chan++) { 7039 if (clear_mask & (1 << chan)) { 7040 sinfo->rx[chan].pkts_f_api = 0; 7041 sinfo->rx[chan].pkts_f_netif = 0; 7042 sinfo->rx[chan].pkts_m_api = 0; 7043 sinfo->rx[chan].pkts_m_netif = 0; 7044 sinfo->rx[chan].pkts_d_no_skb = 0; 7045 sinfo->rx[chan].pkts_d_no_match = 0; 7046 sinfo->rx[chan].pkts_d_unkn_netif = 0; 7047 sinfo->rx[chan].pkts_d_unkn_dest = 0; 7048 sinfo->rx[chan].pkts_d_no_api_buf = 0; 7049 sinfo->rx[chan].sync_err = 0; 7050 sinfo->rx[chan].sync_retry = 0; 7051 sinfo->rx[chan].sync_maxloop = 0; 7052 } 7053 } 7054 7055 return count; 7056 } 7057 7058 struct file_operations bkn_proc_dstats_file_ops = { 7059 owner: THIS_MODULE, 7060 open: bkn_proc_dstats_open, 7061 read: seq_read, 7062 llseek: seq_lseek, 7063 write: bkn_proc_dstats_write, 7064 release: single_release, 7065 }; 7066 7067 static int 7068 bkn_proc_init(void) 7069 { 7070 struct proc_dir_entry *entry; 7071 7072 PROC_CREATE(entry, "link", 0666, bkn_proc_root, &bkn_proc_link_file_ops); 7073 if (entry == NULL) { 7074 return -1; 7075 } 7076 PROC_CREATE(entry, "rate", 0666, bkn_proc_root, &bkn_proc_rate_file_ops); 7077 if (entry == NULL) { 7078 return -1; 7079 } 7080 PROC_CREATE(entry, "dma", 0, bkn_proc_root, &bkn_seq_dma_file_ops); 7081 if (entry == NULL) { 7082 return -1; 7083 } 7084 PROC_CREATE(entry, "debug", 0666, bkn_proc_root, &bkn_proc_debug_file_ops); 7085 if (entry == NULL) { 7086 return -1; 7087 } 7088 PROC_CREATE(entry, "stats", 0666, bkn_proc_root, &bkn_proc_stats_file_ops); 7089 if (entry == NULL) { 7090 return -1; 7091 } 7092 PROC_CREATE(entry, "dstats", 0666, bkn_proc_root, &bkn_proc_dstats_file_ops); 7093 if (entry == NULL) { 7094 return -1; 7095 } 7096 7097 return 0; 7098 } 7099 7100 static int 7101 bkn_proc_cleanup(void) 7102 { 7103 remove_proc_entry("link", bkn_proc_root); 7104 remove_proc_entry("rate", bkn_proc_root); 7105 remove_proc_entry("dma", bkn_proc_root); 7106 remove_proc_entry("debug", bkn_proc_root); 7107 remove_proc_entry("stats", bkn_proc_root); 7108 remove_proc_entry("dstats", bkn_proc_root); 7109 return 0; 7110 } 7111 7112 /* 7113 * Generic module functions 7114 */ 7115 7116 static int 7117 _pprint(void) 7118 { 7119 pprintf("Broadcom BCM KNET Linux Network Driver\n"); 7120 7121 return 0; 7122 } 7123 7124 static int 7125 bkn_knet_dma_info(kcom_msg_dma_info_t *kmsg, int len) 7126 { 7127 bkn_switch_info_t *sinfo; 7128 bkn_dcb_chain_t *dcb_chain, *dcb_chain_end; 7129 unsigned long flags; 7130 int chan; 7131 uint64_t dcb_dma; 7132 int woffset; 7133 7134 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7135 7136 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7137 if (sinfo == NULL) { 7138 kmsg->hdr.status = KCOM_E_PARAM; 7139 return sizeof(kcom_msg_hdr_t); 7140 } 7141 7142 dcb_chain = kmalloc(sizeof(*dcb_chain), GFP_KERNEL); 7143 if (dcb_chain == NULL) { 7144 gprintk("Fatal error: No memory for dcb_chain\n"); 7145 kmsg->hdr.status = KCOM_E_RESOURCE; 7146 return sizeof(kcom_msg_hdr_t); 7147 } 7148 memset(dcb_chain, 0, sizeof(*dcb_chain)); 7149 dcb_chain->dcb_cnt = kmsg->dma_info.cnt; 7150 dcb_chain->dcb_dma = kmsg->dma_info.data.dcb_start; 7151 dcb_chain->dcb_mem = kernel_bde->p2l(sinfo->dev_no, 7152 (sal_paddr_t)dcb_chain->dcb_dma); 7153 7154 if (kmsg->dma_info.type == KCOM_DMA_INFO_T_TX_DCB) { 7155 spin_lock_irqsave(&sinfo->lock, flags); 7156 7157 /* Hold back packets from kernel */ 7158 bkn_suspend_tx(sinfo); 7159 7160 list_add_tail(&dcb_chain->list, &sinfo->tx.api_dcb_list); 7161 7162 /* Handle for Continuous DMA mode */ 7163 if (CDMA_CH(sinfo, XGS_DMA_TX_CHAN)) { 7164 woffset = (dcb_chain->dcb_cnt - 1) * sinfo->dcb_wsize + 1; 7165 if (sinfo->cmic_type == 'x') { 7166 woffset += 1; 7167 } 7168 if ((dcb_chain->dcb_mem[woffset] & ((1 << 18) | (1 << 16))) != 0x50000) { 7169 gprintk("Not suitable API DCB chain for Continuous DMA mode\n"); 7170 list_del(&dcb_chain->list); 7171 kfree(dcb_chain); 7172 kmsg->hdr.status = KCOM_E_PARAM; 7173 spin_unlock_irqrestore(&sinfo->lock, flags); 7174 return sizeof(kcom_msg_hdr_t); 7175 } 7176 dcb_chain_end = sinfo->tx.api_dcb_chain_end; 7177 if (dcb_chain_end != NULL) { 7178 /* Stitch this chain */ 7179 woffset = (dcb_chain_end->dcb_cnt - 1) * sinfo->dcb_wsize; 7180 dcb_chain_end->dcb_mem[woffset] = dcb_chain->dcb_dma; 7181 if (sinfo->cmic_type == 'x') { 7182 dcb_chain_end->dcb_mem[woffset + 1] = DMA_TO_BUS_HI(dcb_chain->dcb_dma >> 32); 7183 } 7184 MEMORY_BARRIER; 7185 } 7186 sinfo->tx.api_dcb_chain_end = dcb_chain; 7187 if (sinfo->tx.api_active) { 7188 /* Set new halt location */ 7189 woffset = (dcb_chain->dcb_cnt - 1) * sinfo->dcb_wsize; 7190 dcb_dma = dcb_chain->dcb_dma + woffset * sizeof(uint32_t); 7191 /* DMA run to the new halt location */ 7192 bkn_cdma_goto(sinfo, XGS_DMA_TX_CHAN, dcb_dma); 7193 } 7194 } 7195 7196 if (sinfo->tx.free == MAX_TX_DCBS && 7197 !sinfo->tx.api_active && 7198 !sinfo->basedev_suspended) { 7199 bkn_api_tx(sinfo); 7200 } 7201 7202 spin_unlock_irqrestore(&sinfo->lock, flags); 7203 } else if (kmsg->dma_info.type == KCOM_DMA_INFO_T_RX_DCB) { 7204 spin_lock_irqsave(&sinfo->lock, flags); 7205 7206 chan = kmsg->dma_info.chan - 1; 7207 if ((chan < 0) || (chan > sinfo->rx_chans)) { 7208 gprintk("Invalid RX DMA channel specified: %d\n", 7209 kmsg->dma_info.chan); 7210 kmsg->hdr.status = KCOM_E_PARAM; 7211 spin_unlock_irqrestore(&sinfo->lock, flags); 7212 return sizeof(kcom_msg_hdr_t); 7213 } 7214 7215 list_add_tail(&dcb_chain->list, &sinfo->rx[chan].api_dcb_list); 7216 7217 /* Handle for Continuous DMA mode */ 7218 if (CDMA_CH(sinfo, XGS_DMA_RX_CHAN + chan)) { 7219 woffset = (dcb_chain->dcb_cnt - 1) * sinfo->dcb_wsize + 1; 7220 if (sinfo->cmic_type == 'x') { 7221 woffset += 1; 7222 } 7223 if ((dcb_chain->dcb_mem[woffset] & ((1 << 18) | (1 << 16))) != 0x50000) { 7224 gprintk("Not suitable API DCB chain for Continuous DMA mode\n"); 7225 list_del(&dcb_chain->list); 7226 kfree(dcb_chain); 7227 kmsg->hdr.status = KCOM_E_PARAM; 7228 spin_unlock_irqrestore(&sinfo->lock, flags); 7229 return sizeof(kcom_msg_hdr_t); 7230 } 7231 dcb_chain_end = sinfo->rx[chan].api_dcb_chain_end; 7232 if (dcb_chain_end != NULL) { 7233 /* Stitch this chain */ 7234 woffset = (dcb_chain_end->dcb_cnt - 1) * sinfo->dcb_wsize; 7235 dcb_chain_end->dcb_mem[woffset] = dcb_chain->dcb_dma; 7236 if (sinfo->cmic_type == 'x') { 7237 dcb_chain_end->dcb_mem[woffset + 1] = DMA_TO_BUS_HI(dcb_chain->dcb_dma >> 32); 7238 } 7239 MEMORY_BARRIER; 7240 } 7241 sinfo->rx[chan].api_dcb_chain_end = dcb_chain; 7242 if (!sinfo->rx[chan].use_rx_skb) { 7243 /* Set new halt location */ 7244 woffset = (dcb_chain->dcb_cnt - 1) * sinfo->dcb_wsize; 7245 dcb_dma = dcb_chain->dcb_dma + woffset * sizeof(uint32_t); 7246 /* DMA run to the new halt location */ 7247 bkn_cdma_goto(sinfo, XGS_DMA_RX_CHAN + chan, dcb_dma); 7248 } 7249 } 7250 7251 if (sinfo->rx[chan].api_active == 0) { 7252 bkn_api_rx_restart(sinfo, chan); 7253 } 7254 7255 spin_unlock_irqrestore(&sinfo->lock, flags); 7256 } else { 7257 DBG_DCB(("Unknown DCB_INFO type (%d).\n", kmsg->dma_info.type)); 7258 kfree(dcb_chain); 7259 kmsg->hdr.status = KCOM_E_PARAM; 7260 } 7261 7262 return sizeof(kcom_msg_hdr_t); 7263 } 7264 7265 static int 7266 bkn_create_inst(uint32 inst_id) 7267 { 7268 bkn_switch_info_t *sinfo; 7269 bkn_evt_resource_t *evt; 7270 unsigned long flags; 7271 int i, evt_idx = -1; 7272 7273 /* multiple instance mode */ 7274 for (i = 0; i < kernel_bde->num_devices(BDE_ALL_DEVICES); i++) { 7275 evt = &_bkn_evt[i]; 7276 if (evt->inst_id == inst_id) { 7277 evt_idx = i; 7278 DBG_INST(("%s evt_idx %d inst_id 0x%x\n",__FUNCTION__, i, inst_id)); 7279 break; 7280 } 7281 if ((_bkn_multi_inst == 0) || (evt->inst_id == 0)) { 7282 _bkn_multi_inst ++; 7283 evt_idx = i; 7284 init_waitqueue_head(&evt->evt_wq); 7285 evt->inst_id = inst_id; 7286 DBG_INST(("%s evt_idx %d inst_id 0x%x\n",__FUNCTION__, i, inst_id)); 7287 break; 7288 } 7289 } 7290 7291 if (evt_idx == -1) { 7292 DBG_WARN(("Run out the event queue resource !\n")); 7293 return -1; 7294 } 7295 for (i = 0; i < kernel_bde->num_devices(BDE_ALL_DEVICES); i++) { 7296 if (inst_id & (1 << i)) { 7297 sinfo = bkn_sinfo_from_unit(i); 7298 spin_lock_irqsave(&sinfo->lock, flags); 7299 sinfo->evt_idx = evt_idx; 7300 spin_unlock_irqrestore(&sinfo->lock, flags); 7301 DBG_INST(("%s d(%d) evt_idx %d \n",__FUNCTION__, i, evt_idx)); 7302 } 7303 } 7304 return 0; 7305 } 7306 7307 /* 7308 * Device reprobe driven by application to check if new device is probed or 7309 * existed device is changed after inserting KNET module. 7310 */ 7311 static int 7312 bkn_knet_dev_reprobe(void) 7313 { 7314 bkn_switch_info_t *sinfo; 7315 int i; 7316 7317 for (i = 0; i < kernel_bde->num_devices(BDE_ALL_DEVICES); i++) { 7318 sinfo = bkn_sinfo_from_unit(i); 7319 if (sinfo == NULL) { 7320 /* New device found after re-probe. */ 7321 if (bkn_knet_dev_init(i) < 0) { 7322 return -1; 7323 } 7324 } else { 7325 /* Existed device reinit after re-probe. */ 7326 if (bkn_knet_dev_reinit(i) < 0) { 7327 return -1; 7328 } 7329 } 7330 } 7331 return 0; 7332 } 7333 7334 /* Assign the inst_id and evt_idx */ 7335 static int 7336 bkn_knet_dev_inst_set(kcom_msg_reprobe_t *kmsg) 7337 { 7338 bkn_switch_info_t *sinfo; 7339 int d = kmsg->hdr.unit; 7340 uint32 inst = 0; 7341 unsigned long flags; 7342 struct list_head *list; 7343 7344 sinfo = bkn_sinfo_from_unit(d); 7345 lkbde_dev_instid_get(d, &inst); 7346 7347 spin_lock_irqsave(&sinfo->lock, flags); 7348 if (sinfo->inst_id != inst) { 7349 /* Instance database changed, reinit the inst_id */ 7350 sinfo->inst_id = 0; 7351 sinfo->evt_idx = -1; 7352 } 7353 spin_unlock_irqrestore(&sinfo->lock, flags); 7354 7355 if (inst) { 7356 if (sinfo->inst_id == 0) { 7357 if (bkn_create_inst(inst) != 0) { 7358 return -1; 7359 } 7360 } 7361 spin_lock_irqsave(&sinfo->lock, flags); 7362 sinfo->inst_id = inst; 7363 spin_unlock_irqrestore(&sinfo->lock, flags); 7364 } else { 7365 /* legacy mode */ 7366 list_for_each(list, &_sinfo_list) { 7367 sinfo = (bkn_switch_info_t *)list; 7368 spin_lock_irqsave(&sinfo->lock, flags); 7369 sinfo->evt_idx = 0; 7370 sinfo->inst_id = 0; 7371 spin_unlock_irqrestore(&sinfo->lock, flags); 7372 } 7373 } 7374 return 0; 7375 } 7376 7377 static int 7378 bkn_knet_version(kcom_msg_version_t *kmsg, int len) 7379 { 7380 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7381 kmsg->version = KCOM_VERSION; 7382 kmsg->netif_max = KCOM_NETIF_MAX; 7383 kmsg->filter_max = KCOM_FILTER_MAX; 7384 7385 return sizeof(kcom_msg_version_t); 7386 } 7387 7388 static int 7389 bkn_knet_hw_reset(kcom_msg_hw_reset_t *kmsg, int len) 7390 { 7391 bkn_switch_info_t *sinfo; 7392 unsigned long flags; 7393 int chan; 7394 7395 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7396 7397 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7398 if (sinfo == NULL) { 7399 kmsg->hdr.status = KCOM_E_PARAM; 7400 return sizeof(kcom_msg_hdr_t); 7401 } 7402 7403 spin_lock_irqsave(&sinfo->lock, flags); 7404 7405 if (kmsg->channels == 0) { 7406 /* Clean all if no channels specified */ 7407 bkn_dma_abort(sinfo); 7408 bkn_clean_dcbs(sinfo); 7409 skb_queue_purge(&sinfo->tx_ptp_queue); 7410 } else { 7411 if (kmsg->channels & (1 << XGS_DMA_TX_CHAN)) { 7412 bkn_dma_abort_tx(sinfo); 7413 bkn_clean_tx_dcbs(sinfo); 7414 skb_queue_purge(&sinfo->tx_ptp_queue); 7415 } 7416 for (chan = 0; chan < sinfo->rx_chans; chan++) { 7417 if (kmsg->channels & (1 << (XGS_DMA_RX_CHAN + chan))) { 7418 bkn_dma_abort_rx(sinfo, chan); 7419 bkn_clean_rx_dcbs(sinfo, chan); 7420 } 7421 } 7422 } 7423 7424 spin_unlock_irqrestore(&sinfo->lock, flags); 7425 7426 return sizeof(kcom_msg_hdr_t); 7427 } 7428 7429 static int 7430 bkn_knet_hw_init(kcom_msg_hw_init_t *kmsg, int len) 7431 { 7432 bkn_switch_info_t *sinfo; 7433 uint32_t dev_type; 7434 unsigned long flags; 7435 7436 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7437 7438 DBG_DCB(("DCB size %d, type %d\n", kmsg->dcb_size, kmsg->dcb_type)); 7439 7440 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7441 if (sinfo == NULL) { 7442 kmsg->hdr.status = KCOM_E_PARAM; 7443 return sizeof(kcom_msg_hdr_t); 7444 } 7445 7446 spin_lock_irqsave(&sinfo->lock, flags); 7447 7448 sinfo->cmic_type = kmsg->cmic_type; 7449 sinfo->dcb_type = kmsg->dcb_type; 7450 sinfo->dcb_wsize = BYTES2WORDS(kmsg->dcb_size); 7451 sinfo->pkt_hdr_size = kmsg->pkt_hdr_size; 7452 sinfo->dma_hi = kmsg->dma_hi; 7453 sinfo->rx_chans = sinfo->cmic_type == 'x' ? NUM_CMICX_RX_CHAN : NUM_CMICM_RX_CHAN; 7454 if (sinfo->rx_chans > NUM_RX_CHAN) { 7455 sinfo->rx_chans = NUM_RX_CHAN; 7456 } 7457 7458 DBG_DUNE(("CMIC:%c DCB:%d WSIZE:%d DMA HI: 0x%08x HDR size: %d\n", 7459 sinfo->cmic_type, sinfo->dcb_type, sinfo->dcb_wsize, 7460 sinfo->dma_hi, sinfo->pkt_hdr_size)); 7461 7462 /* Config Continuous DMA mode */ 7463 sinfo->cdma_channels = kmsg->cdma_channels & ~(~0 << (sinfo->rx_chans + 1)); 7464 7465 /* Ensure 32-bit PCI DMA is mapped properly on 64-bit platforms */ 7466 dev_type = kernel_bde->get_dev_type(sinfo->dev_no); 7467 if (dev_type & BDE_PCI_DEV_TYPE && sinfo->cmic_type != 'x') { 7468 if (pci_set_dma_mask(sinfo->pdev, 0xffffffff)) { 7469 spin_unlock_irqrestore(&sinfo->lock, flags); 7470 gprintk("No suitable DMA available for SKBs\n"); 7471 kmsg->hdr.status = KCOM_E_RESOURCE; 7472 return sizeof(kcom_msg_hdr_t); 7473 } 7474 } 7475 7476 /* First time called we need to allocate DCBs */ 7477 if (sinfo->dcb_mem == NULL) { 7478 if (bkn_alloc_dcbs(sinfo) < 0) { 7479 spin_unlock_irqrestore(&sinfo->lock, flags); 7480 kmsg->hdr.status = KCOM_E_RESOURCE; 7481 return sizeof(kcom_msg_hdr_t); 7482 } 7483 } 7484 7485 /* Ensure that we restart properly */ 7486 bkn_dma_abort(sinfo); 7487 bkn_clean_dcbs(sinfo); 7488 7489 if (basedev_suspend) { 7490 if (!netif_running(sinfo->dev)) { 7491 sinfo->basedev_suspended = 1; 7492 } 7493 } 7494 7495 /* Ensure all interrupts are disabled, e.g. if warmbooting */ 7496 dev_irq_mask_set(sinfo, 0); 7497 7498 /* Register interrupt handler */ 7499 kernel_bde->interrupt_connect(sinfo->dev_no | LKBDE_ISR2_DEV, 7500 bkn_isr, sinfo); 7501 7502 /* Init DCBs */ 7503 bkn_init_dcbs(sinfo); 7504 7505 bkn_dma_init(sinfo); 7506 7507 spin_unlock_irqrestore(&sinfo->lock, flags); 7508 7509 return sizeof(kcom_msg_hdr_t); 7510 } 7511 7512 static int 7513 bkn_knet_detach(kcom_msg_detach_t *kmsg, int len) 7514 { 7515 bkn_switch_info_t *sinfo; 7516 unsigned long flags; 7517 bkn_evt_resource_t *evt; 7518 7519 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7520 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7521 if (sinfo == NULL) { 7522 kmsg->hdr.status = KCOM_E_PARAM; 7523 return sizeof(kcom_msg_hdr_t); 7524 } 7525 7526 spin_lock_irqsave(&sinfo->lock, flags); 7527 7528 /* Create dummy event to unblock pending IOCTL */ 7529 sinfo->dma_events |= KCOM_DMA_INFO_F_TX_DONE; 7530 evt = &_bkn_evt[sinfo->evt_idx]; 7531 evt->evt_wq_put++; 7532 wake_up_interruptible(&evt->evt_wq); 7533 7534 spin_unlock_irqrestore(&sinfo->lock, flags); 7535 7536 /* Ensure that we return a valid unit number */ 7537 kmsg->hdr.unit = sinfo->dev_no; 7538 7539 return sizeof(kcom_msg_detach_t); 7540 } 7541 7542 static int 7543 bkn_knet_reprobe(kcom_msg_reprobe_t *kmsg, int len) 7544 { 7545 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7546 7547 /* Support pci hot plug and multiple instance */ 7548 if ((bkn_knet_dev_reprobe() < 0) || 7549 (bkn_knet_dev_inst_set(kmsg) < 0)) { 7550 kmsg->hdr.status = KCOM_E_RESOURCE; 7551 return sizeof(kcom_msg_reprobe_t); 7552 } 7553 7554 return sizeof(kcom_msg_reprobe_t); 7555 } 7556 7557 static int 7558 bkn_knet_netif_create(kcom_msg_netif_create_t *kmsg, int len) 7559 { 7560 bkn_switch_info_t *sinfo; 7561 struct net_device *dev; 7562 struct list_head *list; 7563 bkn_priv_t *priv, *lpriv; 7564 unsigned long flags; 7565 int found, id; 7566 uint8 *ma; 7567 7568 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7569 7570 switch (kmsg->netif.type) { 7571 case KCOM_NETIF_T_VLAN: 7572 case KCOM_NETIF_T_PORT: 7573 case KCOM_NETIF_T_META: 7574 break; 7575 default: 7576 kmsg->hdr.status = KCOM_E_PARAM; 7577 return sizeof(kcom_msg_hdr_t); 7578 } 7579 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7580 if (sinfo == NULL) { 7581 kmsg->hdr.status = KCOM_E_PARAM; 7582 return sizeof(kcom_msg_hdr_t); 7583 } 7584 ma = kmsg->netif.macaddr; 7585 if ((ma[0] | ma[1] | ma[2] | ma[3] | ma[4] | ma[5]) == 0) { 7586 bkn_dev_mac[5]++; 7587 ma = bkn_dev_mac; 7588 } 7589 if ((dev = bkn_init_ndev(ma, kmsg->netif.name)) == NULL) { 7590 kmsg->hdr.status = KCOM_E_RESOURCE; 7591 return sizeof(kcom_msg_hdr_t); 7592 } 7593 priv = netdev_priv(dev); 7594 priv->dev = dev; 7595 priv->sinfo = sinfo; 7596 priv->type = kmsg->netif.type; 7597 priv->vlan = kmsg->netif.vlan; 7598 if (priv->type == KCOM_NETIF_T_PORT) { 7599 priv->port = kmsg->netif.port; 7600 if (device_is_dpp(sinfo)) { 7601 memcpy(priv->itmh, kmsg->netif.itmh, 4); 7602 } else if (device_is_dnx(sinfo)) { 7603 memcpy(priv->system_headers, kmsg->netif.system_headers, kmsg->netif.system_headers_size); 7604 priv->system_headers_size = kmsg->netif.system_headers_size; 7605 } 7606 priv->qnum = kmsg->netif.qnum; 7607 } else { 7608 if (device_is_sand(sinfo)) { 7609 if (device_is_dpp(sinfo)) { 7610 priv->port = kmsg->netif.port; 7611 priv->qnum = kmsg->netif.qnum; 7612 }else if (device_is_dnx(sinfo)) { 7613 memcpy(priv->system_headers, kmsg->netif.system_headers, kmsg->netif.system_headers_size); 7614 priv->system_headers_size = kmsg->netif.system_headers_size; 7615 } 7616 } 7617 else { 7618 priv->port = -1; 7619 } 7620 } 7621 priv->flags = kmsg->netif.flags; 7622 priv->cb_user_data = kmsg->netif.cb_user_data; 7623 7624 /* Force RCPU encapsulation if rcpu_mode */ 7625 if (rcpu_mode) { 7626 priv->flags |= KCOM_NETIF_F_RCPU_ENCAP; 7627 DBG_RCPU(("RCPU auto-enabled\n")); 7628 } 7629 7630 /* Prevent (incorrect) compiler warning */ 7631 lpriv = NULL; 7632 7633 spin_lock_irqsave(&sinfo->lock, flags); 7634 7635 /* 7636 * We insert network interfaces sorted by ID. 7637 * In case an interface is destroyed, we reuse the ID 7638 * the next time an interface is created. 7639 */ 7640 found = 0; 7641 id = 1; 7642 list_for_each(list, &sinfo->ndev_list) { 7643 lpriv = (bkn_priv_t *)list; 7644 if (id < lpriv->id) { 7645 found = 1; 7646 break; 7647 } 7648 id = lpriv->id + 1; 7649 } 7650 priv->id = id; 7651 if (found) { 7652 /* Replace previously removed interface */ 7653 list_add_tail(&priv->list, &lpriv->list); 7654 } else { 7655 /* No holes - add to end of list */ 7656 list_add_tail(&priv->list, &sinfo->ndev_list); 7657 } 7658 7659 if (id < sinfo->ndev_max) { 7660 DBG_NDEV(("Add netif ID %d to table\n", id)); 7661 sinfo->ndevs[id] = dev; 7662 } else { 7663 int ndev_max = sinfo->ndev_max + NDEVS_CHUNK; 7664 int size = ndev_max * sizeof(struct net_device *); 7665 void *ndevs = kmalloc(size, GFP_ATOMIC); 7666 if (ndevs != NULL) { 7667 DBG_NDEV(("Reallocate netif table for ID %d\n", id)); 7668 memset(ndevs, 0, size); 7669 if (sinfo->ndevs != NULL) { 7670 size = sinfo->ndev_max * sizeof(struct net_device *); 7671 memcpy(ndevs, sinfo->ndevs, size); 7672 kfree(sinfo->ndevs); 7673 } 7674 sinfo->ndevs = ndevs; 7675 sinfo->ndev_max = ndev_max; 7676 sinfo->ndevs[id] = dev; 7677 } 7678 } 7679 7680 spin_unlock_irqrestore(&sinfo->lock, flags); 7681 7682 DBG_VERB(("Assigned ID %d to Ethernet device %s\n", 7683 priv->id, dev->name)); 7684 7685 kmsg->netif.id = priv->id; 7686 memcpy(kmsg->netif.macaddr, dev->dev_addr, 6); 7687 memcpy(kmsg->netif.name, dev->name, KCOM_NETIF_NAME_MAX - 1); 7688 7689 if (device_is_dnx(sinfo)) { 7690 int idx = 0; 7691 for (idx = 0; idx < priv->system_headers_size; idx++) { 7692 DBG_DUNE(("System Header[%d]: 0x%02x\n", idx, priv->system_headers[idx])); 7693 } 7694 } 7695 7696 return sizeof(*kmsg); 7697 } 7698 7699 static int 7700 bkn_knet_netif_destroy(kcom_msg_netif_destroy_t *kmsg, int len) 7701 { 7702 bkn_switch_info_t *sinfo; 7703 struct net_device *dev; 7704 bkn_priv_t *priv; 7705 struct list_head *list; 7706 unsigned long flags; 7707 int found; 7708 7709 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7710 7711 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7712 if (sinfo == NULL) { 7713 kmsg->hdr.status = KCOM_E_PARAM; 7714 return sizeof(kcom_msg_hdr_t); 7715 } 7716 7717 spin_lock_irqsave(&sinfo->lock, flags); 7718 7719 found = 0; 7720 list_for_each(list, &sinfo->ndev_list) { 7721 priv = (bkn_priv_t *)list; 7722 if (kmsg->hdr.id == priv->id) { 7723 found = 1; 7724 break; 7725 } 7726 } 7727 7728 if (!found) { 7729 spin_unlock_irqrestore(&sinfo->lock, flags); 7730 kmsg->hdr.status = KCOM_E_NOT_FOUND; 7731 return sizeof(kcom_msg_hdr_t); 7732 } 7733 7734 list_del(&priv->list); 7735 7736 if (priv->id < sinfo->ndev_max) { 7737 sinfo->ndevs[priv->id] = NULL; 7738 } 7739 7740 spin_unlock_irqrestore(&sinfo->lock, flags); 7741 7742 dev = priv->dev; 7743 DBG_VERB(("Removing virtual Ethernet device %s (%d).\n", 7744 dev->name, priv->id)); 7745 unregister_netdev(dev); 7746 free_netdev(dev); 7747 7748 return sizeof(kcom_msg_hdr_t); 7749 } 7750 7751 static int 7752 bkn_knet_netif_list(kcom_msg_netif_list_t *kmsg, int len) 7753 { 7754 bkn_switch_info_t *sinfo; 7755 bkn_priv_t *priv; 7756 struct list_head *list; 7757 unsigned long flags; 7758 int idx; 7759 7760 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7761 7762 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7763 if (sinfo == NULL) { 7764 kmsg->hdr.status = KCOM_E_PARAM; 7765 return sizeof(kcom_msg_hdr_t); 7766 } 7767 7768 spin_lock_irqsave(&sinfo->lock, flags); 7769 7770 idx = 0; 7771 list_for_each(list, &sinfo->ndev_list) { 7772 if (idx >= KCOM_NETIF_MAX) { 7773 DBG_WARN(("Too many network interfaces to list (max %d).\n", 7774 KCOM_NETIF_MAX)); 7775 break; 7776 } 7777 priv = (bkn_priv_t *)list; 7778 kmsg->id[idx] = priv->id; 7779 idx++; 7780 } 7781 kmsg->ifcnt = idx; 7782 7783 spin_unlock_irqrestore(&sinfo->lock, flags); 7784 7785 return sizeof(*kmsg) - sizeof(kmsg->id) + (idx * sizeof(kmsg->id[0])); 7786 } 7787 7788 static int 7789 bkn_knet_netif_get(kcom_msg_netif_get_t *kmsg, int len) 7790 { 7791 bkn_switch_info_t *sinfo; 7792 bkn_priv_t *priv; 7793 unsigned long flags; 7794 7795 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7796 7797 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7798 if (sinfo == NULL) { 7799 kmsg->hdr.status = KCOM_E_PARAM; 7800 return sizeof(kcom_msg_hdr_t); 7801 } 7802 7803 spin_lock_irqsave(&sinfo->lock, flags); 7804 7805 priv = bkn_netif_lookup(sinfo, kmsg->hdr.id); 7806 7807 if (priv == NULL) { 7808 spin_unlock_irqrestore(&sinfo->lock, flags); 7809 kmsg->hdr.status = KCOM_E_NOT_FOUND; 7810 return sizeof(kcom_msg_hdr_t); 7811 } 7812 7813 memcpy(kmsg->netif.macaddr, priv->dev->dev_addr, 6); 7814 memcpy(kmsg->netif.name, priv->dev->name, KCOM_NETIF_NAME_MAX - 1); 7815 kmsg->netif.vlan = priv->vlan; 7816 kmsg->netif.type = priv->type; 7817 kmsg->netif.id = priv->id; 7818 kmsg->netif.flags = priv->flags; 7819 7820 if (priv->port < 0) { 7821 kmsg->netif.port = 0; 7822 } else { 7823 kmsg->netif.port = priv->port; 7824 } 7825 kmsg->netif.qnum = priv->qnum; 7826 7827 spin_unlock_irqrestore(&sinfo->lock, flags); 7828 7829 return sizeof(*kmsg); 7830 } 7831 7832 static int 7833 bkn_knet_filter_create(kcom_msg_filter_create_t *kmsg, int len) 7834 { 7835 bkn_switch_info_t *sinfo; 7836 struct list_head *list; 7837 bkn_filter_t *filter, *lfilter; 7838 unsigned long flags; 7839 int found, id; 7840 7841 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7842 7843 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7844 if (sinfo == NULL) { 7845 kmsg->hdr.status = KCOM_E_PARAM; 7846 return sizeof(kcom_msg_hdr_t); 7847 } 7848 7849 switch (kmsg->filter.type) { 7850 case KCOM_FILTER_T_RX_PKT: 7851 break; 7852 default: 7853 kmsg->hdr.status = KCOM_E_PARAM; 7854 return sizeof(kcom_msg_hdr_t); 7855 } 7856 7857 spin_lock_irqsave(&sinfo->lock, flags); 7858 7859 /* 7860 * Find available ID 7861 */ 7862 found = 1; 7863 id = 0; 7864 while (found && ++id < KCOM_FILTER_MAX) { 7865 found = 0; 7866 list_for_each(list, &sinfo->rxpf_list) { 7867 lfilter = (bkn_filter_t *)list; 7868 if (id == lfilter->kf.id) { 7869 found = 1; 7870 break; 7871 } 7872 } 7873 } 7874 if (found) { 7875 /* Too many filters */ 7876 spin_unlock_irqrestore(&sinfo->lock, flags); 7877 kmsg->hdr.status = KCOM_E_RESOURCE; 7878 return sizeof(kcom_msg_hdr_t); 7879 } 7880 7881 filter = kmalloc(sizeof(*filter), GFP_ATOMIC); 7882 if (filter == NULL) { 7883 spin_unlock_irqrestore(&sinfo->lock, flags); 7884 kmsg->hdr.status = KCOM_E_PARAM; 7885 return sizeof(kcom_msg_hdr_t); 7886 } 7887 memset(filter, 0, sizeof(*filter)); 7888 memcpy(&filter->kf, &kmsg->filter, sizeof(filter->kf)); 7889 7890 if (device_is_dnx(sinfo)) { 7891 /* Information to parser Dune system headers */ 7892 sinfo->ftmh_lb_key_ext_size = kmsg->filter.ftmh_lb_key_ext_size; 7893 sinfo->ftmh_stacking_ext_size = kmsg->filter.ftmh_stacking_ext_size; 7894 sinfo->pph_base_size = kmsg->filter.pph_base_size; 7895 memcpy(sinfo->pph_lif_ext_size, kmsg->filter.pph_lif_ext_size, sizeof(sinfo->pph_lif_ext_size)); 7896 sinfo->udh_enable = kmsg->filter.udh_enable; 7897 memcpy(sinfo->udh_length_type, kmsg->filter.udh_length_type, sizeof(sinfo->udh_length_type)); 7898 } 7899 7900 filter->kf.id = id; 7901 7902 /* Add according to priority */ 7903 found = 0; 7904 list_for_each(list, &sinfo->rxpf_list) { 7905 lfilter = (bkn_filter_t *)list; 7906 if (filter->kf.priority < lfilter->kf.priority) { 7907 list_add_tail(&filter->list, &lfilter->list); 7908 found = 1; 7909 break; 7910 } 7911 } 7912 if (!found) { 7913 list_add_tail(&filter->list, &sinfo->rxpf_list); 7914 } 7915 7916 kmsg->filter.id = filter->kf.id; 7917 7918 spin_unlock_irqrestore(&sinfo->lock, flags); 7919 7920 DBG_VERB(("Created filter ID %d (%s).\n", 7921 filter->kf.id, filter->kf.desc)); 7922 7923 if (device_is_dnx(sinfo)) { 7924 int idx, wsize; 7925 wsize = BYTES2WORDS(filter->kf.oob_data_size + filter->kf.pkt_data_size); 7926 DBG_DUNE(("Filter: oob_data_size = %d pkt_data_size=%d wsize %d\n", filter->kf.oob_data_size, filter->kf.pkt_data_size, wsize)); 7927 for (idx = 0; idx < wsize; idx++) 7928 { 7929 DBG_DUNE(("OOB[%d]: 0x%08x [0x%08x]\n", idx, filter->kf.data.w[idx], filter->kf.mask.w[idx])); 7930 } 7931 DBG_DUNE(("DNX system headers parameters:LB_KEY_EXT %d, STK_EXT %d, PPH_BASE %d, LIF_EXT %d %d %d, UDH_ENA %d, %d %d %d %d\n", 7932 sinfo->ftmh_lb_key_ext_size, sinfo->ftmh_stacking_ext_size, sinfo->pph_base_size, 7933 sinfo->pph_lif_ext_size[1],sinfo->pph_lif_ext_size[2], sinfo->pph_lif_ext_size[3], 7934 sinfo->udh_enable, sinfo->udh_length_type[0], sinfo->udh_length_type[1], sinfo->udh_length_type[2], sinfo->udh_length_type[3])); 7935 } 7936 7937 return len; 7938 } 7939 7940 static int 7941 bkn_knet_filter_destroy(kcom_msg_filter_destroy_t *kmsg, int len) 7942 { 7943 bkn_switch_info_t *sinfo; 7944 bkn_filter_t *filter; 7945 struct list_head *list; 7946 unsigned long flags; 7947 int found; 7948 7949 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7950 7951 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7952 if (sinfo == NULL) { 7953 kmsg->hdr.status = KCOM_E_PARAM; 7954 return sizeof(kcom_msg_hdr_t); 7955 } 7956 7957 spin_lock_irqsave(&sinfo->lock, flags); 7958 7959 found = 0; 7960 list_for_each(list, &sinfo->rxpf_list) { 7961 filter = (bkn_filter_t *)list; 7962 if (kmsg->hdr.id == filter->kf.id) { 7963 found = 1; 7964 break; 7965 } 7966 } 7967 7968 if (!found) { 7969 spin_unlock_irqrestore(&sinfo->lock, flags); 7970 kmsg->hdr.status = KCOM_E_NOT_FOUND; 7971 return sizeof(kcom_msg_hdr_t); 7972 } 7973 7974 list_del(&filter->list); 7975 7976 spin_unlock_irqrestore(&sinfo->lock, flags); 7977 7978 DBG_VERB(("Removing filter ID %d.\n", filter->kf.id)); 7979 kfree(filter); 7980 7981 return sizeof(kcom_msg_hdr_t); 7982 } 7983 7984 static int 7985 bkn_knet_filter_list(kcom_msg_filter_list_t *kmsg, int len) 7986 { 7987 bkn_switch_info_t *sinfo; 7988 bkn_filter_t *filter; 7989 struct list_head *list; 7990 unsigned long flags; 7991 int idx; 7992 7993 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 7994 7995 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 7996 if (sinfo == NULL) { 7997 kmsg->hdr.status = KCOM_E_PARAM; 7998 return sizeof(kcom_msg_hdr_t); 7999 } 8000 8001 spin_lock_irqsave(&sinfo->lock, flags); 8002 8003 idx = 0; 8004 list_for_each(list, &sinfo->rxpf_list) { 8005 if (idx >= KCOM_FILTER_MAX) { 8006 DBG_WARN(("Too many filters to list (max %d).\n", 8007 KCOM_FILTER_MAX)); 8008 break; 8009 } 8010 filter = (bkn_filter_t *)list; 8011 kmsg->id[idx] = filter->kf.id; 8012 idx++; 8013 } 8014 kmsg->fcnt = idx; 8015 8016 spin_unlock_irqrestore(&sinfo->lock, flags); 8017 8018 return sizeof(*kmsg) - sizeof(kmsg->id) + (idx * sizeof(kmsg->id[0])); 8019 } 8020 8021 static int 8022 bkn_knet_filter_get(kcom_msg_filter_get_t *kmsg, int len) 8023 { 8024 bkn_switch_info_t *sinfo; 8025 bkn_filter_t *filter; 8026 struct list_head *list; 8027 unsigned long flags; 8028 int found; 8029 8030 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 8031 8032 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 8033 if (sinfo == NULL) { 8034 kmsg->hdr.status = KCOM_E_PARAM; 8035 return sizeof(kcom_msg_hdr_t); 8036 } 8037 8038 spin_lock_irqsave(&sinfo->lock, flags); 8039 8040 found = 0; 8041 list_for_each(list, &sinfo->rxpf_list) { 8042 filter = (bkn_filter_t *)list; 8043 if (kmsg->hdr.id == filter->kf.id) { 8044 found = 1; 8045 break; 8046 } 8047 } 8048 8049 if (!found) { 8050 spin_unlock_irqrestore(&sinfo->lock, flags); 8051 kmsg->hdr.status = KCOM_E_NOT_FOUND; 8052 return sizeof(kcom_msg_hdr_t); 8053 } 8054 8055 memcpy(&kmsg->filter, &filter->kf, sizeof(kmsg->filter)); 8056 8057 spin_unlock_irqrestore(&sinfo->lock, flags); 8058 8059 return sizeof(*kmsg); 8060 } 8061 8062 static int 8063 bkn_knet_dbg_pkt_set(kcom_msg_dbg_pkt_set_t *kmsg, int len) 8064 { 8065 dbg_pkt_enable = kmsg->enable; 8066 return sizeof(kcom_msg_dbg_pkt_set_t); 8067 } 8068 8069 static int 8070 bkn_knet_dbg_pkt_get(kcom_msg_dbg_pkt_get_t *kmsg, int len) 8071 { 8072 kmsg->value = dbg_pkt_enable; 8073 return sizeof(kcom_msg_dbg_pkt_get_t); 8074 } 8075 8076 static int 8077 bkn_knet_wb_cleanup(kcom_msg_wb_cleanup_t *kmsg, int len) 8078 { 8079 bkn_switch_info_t *sinfo; 8080 bkn_dcb_chain_t *dcb_chain; 8081 unsigned long flags; 8082 int chan; 8083 8084 kmsg->hdr.type = KCOM_MSG_TYPE_RSP; 8085 8086 sinfo = bkn_sinfo_from_unit(kmsg->hdr.unit); 8087 if (sinfo == NULL) { 8088 kmsg->hdr.status = KCOM_E_PARAM; 8089 return sizeof(kcom_msg_hdr_t); 8090 } 8091 8092 spin_lock_irqsave(&sinfo->lock, flags); 8093 8094 for (chan = 0; chan < sinfo->rx_chans; chan++) { 8095 if (sinfo->rx[chan].api_dcb_chain) { 8096 DBG_DCB_RX(("Freeing active Rx%d DCB chain.\n", chan)); 8097 kfree(sinfo->rx[chan].api_dcb_chain); 8098 sinfo->rx[chan].api_dcb_chain = NULL; 8099 } 8100 while (!list_empty(&sinfo->rx[chan].api_dcb_list)) { 8101 dcb_chain = list_entry(sinfo->rx[chan].api_dcb_list.next, 8102 bkn_dcb_chain_t, list); 8103 list_del(&dcb_chain->list); 8104 DBG_DCB_RX(("Freeing Rx%d DCB chain.\n", chan)); 8105 kfree(dcb_chain); 8106 } 8107 sinfo->rx[chan].api_dcb_chain_end = NULL; 8108 sinfo->rx[chan].api_active = 0; 8109 } 8110 8111 spin_unlock_irqrestore(&sinfo->lock, flags); 8112 8113 return sizeof(kcom_msg_hdr_t); 8114 } 8115 8116 static int 8117 bkn_handle_cmd_req(kcom_msg_t *kmsg, int len) 8118 { 8119 /* Silently drop events and unrecognized message types */ 8120 if (kmsg->hdr.type != KCOM_MSG_TYPE_CMD) { 8121 if (kmsg->hdr.opcode == KCOM_M_STRING) { 8122 DBG_VERB(("Debug string: '%s'\n", kmsg->string.val)); 8123 return 0; 8124 } 8125 DBG_WARN(("Unsupported message (type=%d, opcode=%d)\n", 8126 kmsg->hdr.type, kmsg->hdr.opcode)); 8127 return 0; 8128 } 8129 8130 switch (kmsg->hdr.opcode) { 8131 case KCOM_M_DMA_INFO: 8132 DBG_CMD(("KCOM_M_DMA_INFO\n")); 8133 /* Packet buffer */ 8134 len = bkn_knet_dma_info(&kmsg->dma_info, len); 8135 break; 8136 case KCOM_M_VERSION: 8137 DBG_CMD(("KCOM_M_VERSION\n")); 8138 /* Return procotol version */ 8139 len = bkn_knet_version(&kmsg->version, len); 8140 break; 8141 case KCOM_M_HW_RESET: 8142 DBG_CMD(("KCOM_M_HW_RESET\n")); 8143 /* Shut down DMA and release buffers */ 8144 len = bkn_knet_hw_reset(&kmsg->hw_reset, len); 8145 break; 8146 case KCOM_M_HW_INIT: 8147 DBG_CMD(("KCOM_M_HW_INIT\n")); 8148 /* Initialize DMA */ 8149 len = bkn_knet_hw_init(&kmsg->hw_init, len); 8150 break; 8151 case KCOM_M_DETACH: 8152 DBG_CMD(("KCOM_M_DETACH\n")); 8153 /* Detach kernel module */ 8154 len = bkn_knet_detach(&kmsg->detach, len); 8155 break; 8156 case KCOM_M_REPROBE: 8157 DBG_CMD(("KCOM_M_REPROBE\n")); 8158 /* Reprobe device */ 8159 len = bkn_knet_reprobe(&kmsg->reprobe, len); 8160 break; 8161 case KCOM_M_NETIF_CREATE: 8162 DBG_CMD(("KCOM_M_NETIF_CREATE\n")); 8163 /* Create network interface */ 8164 len = bkn_knet_netif_create(&kmsg->netif_create, len); 8165 break; 8166 case KCOM_M_NETIF_DESTROY: 8167 DBG_CMD(("KCOM_M_NETIF_DESTROY\n")); 8168 /* Destroy network interface */ 8169 len = bkn_knet_netif_destroy(&kmsg->netif_destroy, len); 8170 break; 8171 case KCOM_M_NETIF_LIST: 8172 DBG_CMD(("KCOM_M_NETIF_LIST\n")); 8173 /* Return list of IDs of installed network interfaces */ 8174 len = bkn_knet_netif_list(&kmsg->netif_list, len); 8175 break; 8176 case KCOM_M_NETIF_GET: 8177 DBG_CMD(("KCOM_M_NETIF_GET\n")); 8178 /* Return network interface info */ 8179 len = bkn_knet_netif_get(&kmsg->netif_get, len); 8180 break; 8181 case KCOM_M_FILTER_CREATE: 8182 DBG_CMD(("KCOM_M_FILTER_CREATE\n")); 8183 /* Create packet filter */ 8184 len = bkn_knet_filter_create(&kmsg->filter_create, len); 8185 break; 8186 case KCOM_M_FILTER_DESTROY: 8187 DBG_CMD(("KCOM_M_FILTER_DESTROY\n")); 8188 /* Destroy packet filter */ 8189 len = bkn_knet_filter_destroy(&kmsg->filter_destroy, len); 8190 break; 8191 case KCOM_M_FILTER_LIST: 8192 DBG_CMD(("KCOM_M_FILTER_LIST\n")); 8193 /* Return list of IDs of installed packet filters */ 8194 len = bkn_knet_filter_list(&kmsg->filter_list, len); 8195 break; 8196 case KCOM_M_FILTER_GET: 8197 DBG_CMD(("KCOM_M_FILTER_GET\n")); 8198 /* Return packet filter info */ 8199 len = bkn_knet_filter_get(&kmsg->filter_get, len); 8200 break; 8201 case KCOM_M_DBGPKT_SET: 8202 DBG_CMD(("KCOM_M_DBGPKT_SET\n")); 8203 /* Set debugging packet function */ 8204 len = bkn_knet_dbg_pkt_set(&kmsg->dbg_pkt_set, len); 8205 break; 8206 case KCOM_M_DBGPKT_GET: 8207 DBG_CMD(("KCOM_M_DBGPKT_GET\n")); 8208 /* Get debugging packet function info */ 8209 len = bkn_knet_dbg_pkt_get(&kmsg->dbg_pkt_get, len); 8210 break; 8211 case KCOM_M_WB_CLEANUP: 8212 DBG_CMD(("KCOM_M_WB_CLEANUP\n")); 8213 /* Clean up for warmbooting */ 8214 len = bkn_knet_wb_cleanup(&kmsg->wb_cleanup, len); 8215 break; 8216 default: 8217 DBG_WARN(("Unsupported command (type=%d, opcode=%d)\n", 8218 kmsg->hdr.type, kmsg->hdr.opcode)); 8219 kmsg->hdr.opcode = 0; 8220 len = sizeof(kcom_msg_hdr_t); 8221 break; 8222 } 8223 return len; 8224 } 8225 8226 static int 8227 bkn_cmd_thread(void *context) 8228 { 8229 bkn_thread_ctrl_t *tc = (bkn_thread_ctrl_t *)context; 8230 kcom_msg_t kmsg; 8231 unsigned int len, rlen; 8232 8233 bkn_thread_boot(tc); 8234 8235 DBG_VERB(("Command thread starting\n")); 8236 tc->state = 1; 8237 while (!bkn_thread_should_stop(tc)) { 8238 len = sizeof(kmsg); 8239 tc->state = 2; 8240 if (PROXY_RECV(KCOM_CHAN_KNET, &kmsg, &len) >= 0) { 8241 DBG_VERB(("Received %d bytes from KCOM_CHAN_CMD\n", len)); 8242 tc->state = 3; 8243 rlen = bkn_handle_cmd_req(&kmsg, len); 8244 tc->state = 4; 8245 if (rlen > 0) { 8246 PROXY_SEND(KCOM_CHAN_KNET, &kmsg, rlen); 8247 } 8248 } else { 8249 /* Thread interrupted */ 8250 bkn_sleep(1); 8251 } 8252 } 8253 DBG_VERB(("Command thread done\n")); 8254 8255 bkn_thread_exit(tc); 8256 return 0; 8257 } 8258 8259 static int 8260 bkn_get_next_dma_event(kcom_msg_dma_info_t *kmsg) 8261 { 8262 static int last_dev_no = 0; 8263 bkn_switch_info_t *sinfo; 8264 unsigned long flags; 8265 int dev_no, dev_evt; 8266 bkn_evt_resource_t *evt; 8267 8268 dev_evt = kmsg->hdr.unit; 8269 sinfo = bkn_sinfo_from_unit(dev_evt); 8270 if (sinfo == NULL) { 8271 /* The device is not probed or initialized yet.*/ 8272 return 0; 8273 } 8274 if (sinfo->evt_idx == -1) { 8275 /* Event queue is not ready yet */ 8276 return 0; 8277 } 8278 8279 DBG_INST(("%s dev %d evt_idx %d\n",__FUNCTION__, dev_evt, sinfo->evt_idx)); 8280 dev_no = last_dev_no; 8281 while (1) { 8282 dev_no++; 8283 sinfo = bkn_sinfo_from_unit(dev_no); 8284 if (sinfo == NULL) { 8285 dev_no = 0; 8286 sinfo = bkn_sinfo_from_unit(dev_no); 8287 } 8288 8289 if (sinfo && (sinfo->inst_id != 0) && 8290 ((sinfo->inst_id & (1 << dev_evt)) == 0)) { 8291 DBG_INST((" %s skip dev(%d)\n",__FUNCTION__,dev_evt)); 8292 continue; 8293 } 8294 8295 if (sinfo && sinfo->dma_events) { 8296 DBG_EVT(("Next DMA events (0x%08x)\n", sinfo->dma_events)); 8297 kmsg->hdr.unit = sinfo->dev_no; 8298 8299 spin_lock_irqsave(&sinfo->lock, flags); 8300 kmsg->dma_info.flags = sinfo->dma_events; 8301 sinfo->dma_events = 0; 8302 spin_unlock_irqrestore(&sinfo->lock, flags); 8303 8304 last_dev_no = dev_no; 8305 break; 8306 } 8307 8308 if (dev_no == last_dev_no) { 8309 evt = &_bkn_evt[sinfo->evt_idx]; 8310 DBG_INST(("wait queue index %d\n",sinfo->evt_idx)); 8311 wait_event_interruptible(evt->evt_wq, 8312 evt->evt_wq_get != evt->evt_wq_put); 8313 DBG_VERB(("Event thread wakeup\n")); 8314 8315 /* Thread interrupted */ 8316 if (signal_pending(current)) { 8317 return 0; 8318 } 8319 8320 evt->evt_wq_get = evt->evt_wq_put; 8321 } 8322 } 8323 return sizeof(*kmsg); 8324 } 8325 8326 static int 8327 bkn_evt_thread(void *context) 8328 { 8329 bkn_thread_ctrl_t *tc = (bkn_thread_ctrl_t *)context; 8330 kcom_msg_dma_info_t kmsg; 8331 int len; 8332 8333 bkn_thread_boot(tc); 8334 8335 memset(&kmsg, 0, sizeof(kmsg)); 8336 kmsg.hdr.type = KCOM_MSG_TYPE_EVT; 8337 kmsg.hdr.opcode = KCOM_M_DMA_INFO; 8338 8339 DBG_VERB(("Event thread starting\n")); 8340 tc->state = 1; 8341 while (!bkn_thread_should_stop(tc)) { 8342 tc->state = 2; 8343 len = bkn_get_next_dma_event(&kmsg); 8344 tc->state = 3; 8345 if (len) { 8346 PROXY_SEND(KCOM_CHAN_KNET, &kmsg, len); 8347 } else { 8348 /* Thread interrupted */ 8349 bkn_sleep(1); 8350 } 8351 } 8352 DBG_VERB(("Event thread done\n")); 8353 8354 bkn_thread_exit(tc); 8355 return 0; 8356 } 8357 8358 static int 8359 _cleanup(void) 8360 { 8361 struct list_head *list; 8362 struct net_device *dev; 8363 bkn_filter_t *filter; 8364 bkn_priv_t *priv; 8365 bkn_switch_info_t *sinfo; 8366 unsigned long flags; 8367 8368 /* Inidicate that we are shutting down */ 8369 module_initialized = 0; 8370 8371 /* Shut down event thread */ 8372 bkn_thread_stop(&bkn_evt_ctrl); 8373 8374 /* Shut down command thread */ 8375 bkn_thread_stop(&bkn_cmd_ctrl); 8376 8377 /* Remove KCOM channel */ 8378 PROXY_SERVICE_DESTROY(KCOM_CHAN_KNET); 8379 8380 bkn_proc_cleanup(); 8381 remove_proc_entry("bcm/knet", NULL); 8382 remove_proc_entry("bcm", NULL); 8383 8384 list_for_each(list, &_sinfo_list) { 8385 sinfo = (bkn_switch_info_t *)list; 8386 8387 del_timer_sync(&sinfo->timer); 8388 del_timer_sync(&sinfo->rxtick); 8389 8390 spin_lock_irqsave(&sinfo->lock, flags); 8391 bkn_dma_abort(sinfo); 8392 dev_irq_mask_set(sinfo, 0); 8393 spin_unlock_irqrestore(&sinfo->lock, flags); 8394 8395 DBG_IRQ(("Unregister ISR.\n")); 8396 kernel_bde->interrupt_disconnect(sinfo->dev_no | LKBDE_ISR2_DEV); 8397 8398 if (use_napi) { 8399 while (sinfo->napi_poll_mode) { 8400 bkn_sleep(1); 8401 } 8402 } 8403 8404 spin_lock_irqsave(&sinfo->lock, flags); 8405 bkn_clean_dcbs(sinfo); 8406 skb_queue_purge(&sinfo->tx_ptp_queue); 8407 spin_unlock_irqrestore(&sinfo->lock, flags); 8408 } 8409 8410 /* Destroy all switch devices */ 8411 while (!list_empty(&_sinfo_list)) { 8412 sinfo = list_entry(_sinfo_list.next, bkn_switch_info_t, list); 8413 8414 /* Destroy all associated Rx packet filters */ 8415 while (!list_empty(&sinfo->rxpf_list)) { 8416 filter = list_entry(sinfo->rxpf_list.next, bkn_filter_t, list); 8417 list_del(&filter->list); 8418 DBG_VERB(("Removing filter ID %d.\n", filter->kf.id)); 8419 kfree(filter); 8420 } 8421 8422 /* Destroy all associated virtual net devices */ 8423 while (!list_empty(&sinfo->ndev_list)) { 8424 priv = list_entry(sinfo->ndev_list.next, bkn_priv_t, list); 8425 list_del(&priv->list); 8426 dev = priv->dev; 8427 DBG_VERB(("Removing virtual Ethernet device %s.\n", dev->name)); 8428 unregister_netdev(dev); 8429 free_netdev(dev); 8430 } 8431 if (sinfo->ndevs != NULL) { 8432 kfree(sinfo->ndevs); 8433 } 8434 8435 /* Destroy base net device */ 8436 if (sinfo->dev) { 8437 DBG_VERB(("Removing Ethernet device %s.\n", sinfo->dev->name)); 8438 unregister_netdev(sinfo->dev); 8439 free_netdev(sinfo->dev); 8440 } 8441 8442 DBG_VERB(("Removing switch device.\n")); 8443 bkn_destroy_sinfo(sinfo); 8444 } 8445 8446 return 0; 8447 } 8448 8449 static int 8450 bkn_knet_dev_reinit(int d) 8451 { 8452 bkn_switch_info_t *sinfo; 8453 uint32 dev_state; 8454 unsigned long flags; 8455 8456 if (lkbde_dev_state_get(d, &dev_state) < 0) { 8457 return -1; 8458 } 8459 DBG_VERB(("%s dev %d dev_state %d\n",__FUNCTION__, d, dev_state)); 8460 if (dev_state == BDE_DEV_STATE_CHANGED) { 8461 sinfo = bkn_sinfo_from_unit(d); 8462 spin_lock_irqsave(&sinfo->lock, flags); 8463 sinfo->base_addr = lkbde_get_dev_virt(d); 8464 sinfo->dma_dev = lkbde_get_dma_dev(d); 8465 sinfo->pdev = lkbde_get_hw_dev(d); 8466 spin_unlock_irqrestore(&sinfo->lock, flags); 8467 8468 dev_state = 0; 8469 lkbde_dev_state_set(d, dev_state); 8470 } 8471 return 0; 8472 } 8473 8474 static int 8475 bkn_knet_dev_init(int d) 8476 { 8477 uint32_t dev_type; 8478 struct net_device *dev; 8479 bkn_switch_info_t *sinfo; 8480 bkn_priv_t *priv; 8481 char *bdev_name; 8482 const ibde_dev_t *bde_dev; 8483 8484 DBG_VERB(("%s dev %d\n",__FUNCTION__, d)); 8485 /* Base network device name */ 8486 bdev_name = "bcm%d"; 8487 if (base_dev_name) { 8488 if (strlen(base_dev_name) < IFNAMSIZ) { 8489 bdev_name = base_dev_name; 8490 } else { 8491 DBG_WARN(("Base device name too long\n")); 8492 } 8493 } 8494 8495 dev_type = kernel_bde->get_dev_type(d); 8496 DBG_VERB(("Found device type 0x%x\n", dev_type)); 8497 if ((dev_type & BDE_SWITCH_DEV_TYPE) == 0) { 8498 DBG_WARN(("Not switch device - skipping\n")); 8499 return 0; 8500 } 8501 switch (dev_type & BDE_DEV_BUS_TYPE_MASK) { 8502 case BDE_PCI_DEV_TYPE: 8503 case BDE_ICS_DEV_TYPE: 8504 case BDE_AXI_DEV_TYPE: 8505 break; 8506 default: 8507 DBG_WARN(("Not PCI/ICS/AXI device - skipping\n")); 8508 return 0; 8509 } 8510 8511 if ((sinfo = bkn_create_sinfo(d)) == NULL) { 8512 _cleanup(); 8513 return -ENOMEM; 8514 } 8515 8516 /* Initialize the cpu_no.*/ 8517 if (dev_type & BDE_AXI_DEV_TYPE) { 8518 /* AXI device type implies the activated iProc iHost */ 8519 sinfo->cpu_no = 1; 8520 } 8521 /* Initialize default RCPU signature */ 8522 if ((bde_dev = kernel_bde->get_dev(d)) != NULL) { 8523 sinfo->rcpu_sig = bde_dev->device & ~0xf; 8524 } 8525 /* Check for override */ 8526 if (rcpu_signature) { 8527 sinfo->rcpu_sig = rcpu_signature; 8528 } 8529 8530 /* Create base virtual net device */ 8531 bkn_dev_mac[5]++; 8532 if ((dev = bkn_init_ndev(bkn_dev_mac, bdev_name)) == NULL) { 8533 _cleanup(); 8534 return -ENOMEM; 8535 } else { 8536 sinfo->dev = dev; 8537 priv = netdev_priv(dev); 8538 priv->dev = dev; 8539 priv->sinfo = sinfo; 8540 priv->vlan = 1; 8541 priv->port = -1; 8542 memset(priv->itmh, 0, sizeof(priv->itmh)); 8543 priv->id = -1; 8544 } 8545 8546 if (use_napi) { 8547 netif_napi_add(dev, &sinfo->napi, bkn_poll, napi_weight); 8548 } 8549 return 0; 8550 } 8551 8552 static int 8553 _init(void) 8554 { 8555 int idx; 8556 int num_dev; 8557 int rv; 8558 bkn_evt_resource_t *evt; 8559 8560 /* Connect to the kernel bde */ 8561 if ((linux_bde_create(NULL, &kernel_bde) < 0) || kernel_bde == NULL) { 8562 return -ENODEV; 8563 } 8564 8565 /* Randomize Lower 3 bytes of the MAC address (TESTING ONLY) */ 8566 get_random_bytes(&bkn_dev_mac[3], 3); 8567 8568 /* Check for user-supplied MAC address (recommended) */ 8569 if (mac_addr != NULL && strlen(mac_addr) == 17) { 8570 for (idx = 0; idx < 6; idx++) { 8571 bkn_dev_mac[idx] = simple_strtoul(&mac_addr[idx*3], NULL, 16); 8572 } 8573 /* Do not allow multicast address */ 8574 bkn_dev_mac[0] &= ~0x01; 8575 } 8576 8577 /* Optional RCPU MAC addresses */ 8578 if (rcpu_dmac != NULL && strlen(rcpu_dmac) == 17) { 8579 for (idx = 0; idx < 6; idx++) { 8580 bkn_rcpu_dmac[idx] = simple_strtoul(&rcpu_dmac[idx*3], NULL, 16); 8581 } 8582 } 8583 if (rcpu_smac != NULL && strlen(rcpu_smac) == 17) { 8584 for (idx = 0; idx < 6; idx++) { 8585 bkn_rcpu_smac[idx] = simple_strtoul(&rcpu_smac[idx*3], NULL, 16); 8586 } 8587 } 8588 8589 /* NAPI implies that base device must be up before we can pass traffic */ 8590 if (use_napi) { 8591 basedev_suspend = 1; 8592 } 8593 8594 num_dev = kernel_bde->num_devices(BDE_ALL_DEVICES); 8595 for (idx = 0; idx < num_dev; idx++) { 8596 rv = bkn_knet_dev_init(idx); 8597 if (rv) { 8598 return rv; 8599 } 8600 } 8601 8602 /* Initialize proc files */ 8603 proc_mkdir("bcm", NULL); 8604 bkn_proc_root = proc_mkdir("bcm/knet", NULL); 8605 8606 bkn_proc_init(); 8607 8608 /* Initialize event queue */ 8609 for (idx = 0; idx < LINUX_BDE_MAX_DEVICES; idx++) { 8610 memset(&_bkn_evt[idx], 0, sizeof(bkn_evt_resource_t)); 8611 } 8612 evt = &_bkn_evt[0]; 8613 init_waitqueue_head(&evt->evt_wq); 8614 8615 if (use_proxy) { 8616 PROXY_SERVICE_CREATE(KCOM_CHAN_KNET, 1, 0); 8617 8618 DBG_VERB(("Starting command thread\n")); 8619 bkn_thread_start(&bkn_cmd_ctrl, "bkncmd", bkn_cmd_thread); 8620 8621 DBG_VERB(("Starting event thread\n")); 8622 bkn_thread_start(&bkn_evt_ctrl, "bknevt", bkn_evt_thread); 8623 } 8624 8625 module_initialized = 1; 8626 8627 return 0; 8628 } 8629 8630 static int 8631 _ioctl(unsigned int cmd, unsigned long arg) 8632 { 8633 bkn_ioctl_t io; 8634 kcom_msg_t kmsg; 8635 8636 if (!module_initialized) { 8637 return -EFAULT; 8638 } 8639 8640 if (copy_from_user(&io, (void*)arg, sizeof(io))) { 8641 return -EFAULT; 8642 } 8643 8644 if (io.len > sizeof(kmsg)) { 8645 return -EINVAL; 8646 } 8647 8648 io.rc = 0; 8649 8650 switch(cmd) { 8651 case 0: 8652 if (io.len > 0) { 8653 if (copy_from_user(&kmsg, (void *)(unsigned long)io.buf, io.len)) { 8654 return -EFAULT; 8655 } 8656 ioctl_cmd++; 8657 io.len = bkn_handle_cmd_req(&kmsg, io.len); 8658 ioctl_cmd--; 8659 } else { 8660 memset(&kmsg, 0, sizeof(kcom_msg_dma_info_t)); 8661 /* 8662 * Retrive the kmsg.hdr.unit from user space. The dma event queue 8663 * selection is based the instance derived from unit. 8664 */ 8665 if (copy_from_user(&kmsg, (void *)(unsigned long)io.buf, sizeof(kmsg))) { 8666 return -EFAULT; 8667 } 8668 kmsg.hdr.type = KCOM_MSG_TYPE_EVT; 8669 kmsg.hdr.opcode = KCOM_M_DMA_INFO; 8670 ioctl_evt++; 8671 io.len = bkn_get_next_dma_event((kcom_msg_dma_info_t *)&kmsg); 8672 ioctl_evt--; 8673 } 8674 if (io.len > 0) { 8675 if (copy_to_user((void *)(unsigned long)io.buf, &kmsg, io.len)) { 8676 return -EFAULT; 8677 } 8678 } 8679 break; 8680 default: 8681 gprintk("Invalid IOCTL"); 8682 io.rc = -1; 8683 break; 8684 } 8685 8686 if (copy_to_user((void*)arg, &io, sizeof(io))) { 8687 return -EFAULT; 8688 } 8689 8690 return 0; 8691 } 8692 8693 static gmodule_t _gmodule = { 8694 name: MODULE_NAME, 8695 major: MODULE_MAJOR, 8696 init: _init, 8697 cleanup: _cleanup, 8698 pprint: _pprint, 8699 ioctl: _ioctl, 8700 open: NULL, 8701 close: NULL, 8702 }; 8703 8704 gmodule_t * 8705 gmodule_get(void) 8706 { 8707 EXPORT_NO_SYMBOLS; 8708 return &_gmodule; 8709 } 8710 8711 /* 8712 * Call-back interfaces for other Linux kernel drivers. 8713 * 8714 * The Rx call-back allows an external module to modify SKB contents 8715 * before it is handed off to the Linux network stack. 8716 * 8717 * The Tx call-back allows an external module to modify SKB contents 8718 * before it is injected inot the switch. 8719 * 8720 * The HW timestamp callbacks invoke an external module to enable, disable 8721 * HW timestamp on a specific port which is indicated while the netif is 8722 * created through KNET API. KNET can also get device-specific SOBMH and 8723 * timestamp for a Tx PTP packet through these callbacks. 8724 */ 8725 8726 int 8727 bkn_rx_skb_cb_register(knet_skb_cb_f rx_cb) 8728 { 8729 if (knet_rx_cb != NULL) { 8730 return -1; 8731 } 8732 knet_rx_cb = rx_cb; 8733 return 0; 8734 } 8735 8736 int 8737 bkn_rx_skb_cb_unregister(knet_skb_cb_f rx_cb) 8738 { 8739 if (rx_cb != NULL && knet_rx_cb != rx_cb) { 8740 return -1; 8741 } 8742 knet_rx_cb = NULL; 8743 return 0; 8744 } 8745 8746 int 8747 bkn_tx_skb_cb_register(knet_skb_cb_f tx_cb) 8748 { 8749 if (knet_tx_cb != NULL) { 8750 return -1; 8751 } 8752 knet_tx_cb = tx_cb; 8753 return 0; 8754 } 8755 8756 int 8757 bkn_tx_skb_cb_unregister(knet_skb_cb_f tx_cb) 8758 { 8759 if (tx_cb != NULL && knet_tx_cb != tx_cb) { 8760 return -1; 8761 } 8762 knet_tx_cb = NULL; 8763 return 0; 8764 } 8765 8766 int 8767 bkn_filter_cb_register(knet_filter_cb_f filter_cb) 8768 { 8769 if (knet_filter_cb != NULL) { 8770 return -1; 8771 } 8772 knet_filter_cb = filter_cb; 8773 return 0; 8774 } 8775 8776 int 8777 bkn_filter_cb_unregister(knet_filter_cb_f filter_cb) 8778 { 8779 if (filter_cb != NULL && knet_filter_cb != filter_cb) { 8780 return -1; 8781 } 8782 knet_filter_cb = NULL; 8783 return 0; 8784 } 8785 8786 int 8787 bkn_hw_tstamp_enable_cb_register(knet_hw_tstamp_enable_cb_f hw_tstamp_enable_cb) 8788 { 8789 if (knet_hw_tstamp_enable_cb != NULL) { 8790 return -1; 8791 } 8792 knet_hw_tstamp_enable_cb = hw_tstamp_enable_cb; 8793 return 0; 8794 } 8795 8796 int 8797 bkn_hw_tstamp_enable_cb_unregister(knet_hw_tstamp_enable_cb_f hw_tstamp_enable_cb) 8798 { 8799 if (hw_tstamp_enable_cb == NULL || 8800 knet_hw_tstamp_enable_cb != hw_tstamp_enable_cb) { 8801 return -1; 8802 } 8803 knet_hw_tstamp_enable_cb = NULL; 8804 return 0; 8805 } 8806 8807 int 8808 bkn_hw_tstamp_disable_cb_register(knet_hw_tstamp_enable_cb_f hw_tstamp_disable_cb) 8809 { 8810 if (knet_hw_tstamp_disable_cb != NULL) { 8811 return -1; 8812 } 8813 knet_hw_tstamp_disable_cb = hw_tstamp_disable_cb; 8814 return 0; 8815 } 8816 8817 int 8818 bkn_hw_tstamp_disable_cb_unregister(knet_hw_tstamp_enable_cb_f hw_tstamp_disable_cb) 8819 { 8820 if (hw_tstamp_disable_cb == NULL || 8821 knet_hw_tstamp_disable_cb != hw_tstamp_disable_cb) { 8822 return -1; 8823 } 8824 knet_hw_tstamp_disable_cb = NULL; 8825 return 0; 8826 } 8827 8828 int 8829 bkn_hw_tstamp_tx_time_get_cb_register(knet_hw_tstamp_tx_time_get_cb_f hw_tstamp_tx_time_get_cb) 8830 { 8831 if (knet_hw_tstamp_tx_time_get_cb != NULL) { 8832 return -1; 8833 } 8834 knet_hw_tstamp_tx_time_get_cb = hw_tstamp_tx_time_get_cb; 8835 return 0; 8836 } 8837 8838 int 8839 bkn_hw_tstamp_tx_time_get_cb_unregister(knet_hw_tstamp_tx_time_get_cb_f hw_tstamp_tx_time_get_cb) 8840 { 8841 if (hw_tstamp_tx_time_get_cb == NULL || 8842 knet_hw_tstamp_tx_time_get_cb != hw_tstamp_tx_time_get_cb) { 8843 return -1; 8844 } 8845 knet_hw_tstamp_tx_time_get_cb = NULL; 8846 return 0; 8847 } 8848 8849 int 8850 bkn_hw_tstamp_tx_meta_get_cb_register(knet_hw_tstamp_tx_meta_get_cb_f hw_tstamp_tx_meta_get_cb) 8851 { 8852 if (knet_hw_tstamp_tx_meta_get_cb != NULL) { 8853 return -1; 8854 } 8855 knet_hw_tstamp_tx_meta_get_cb = hw_tstamp_tx_meta_get_cb; 8856 return 0; 8857 } 8858 8859 int 8860 bkn_hw_tstamp_tx_meta_get_cb_unregister(knet_hw_tstamp_tx_meta_get_cb_f hw_tstamp_tx_meta_get_cb) 8861 { 8862 if (hw_tstamp_tx_meta_get_cb == NULL || 8863 knet_hw_tstamp_tx_meta_get_cb != hw_tstamp_tx_meta_get_cb) { 8864 return -1; 8865 } 8866 knet_hw_tstamp_tx_meta_get_cb = NULL; 8867 return 0; 8868 } 8869 8870 int 8871 bkn_hw_tstamp_ptp_clock_index_cb_register(knet_hw_tstamp_ptp_clock_index_cb_f hw_tstamp_ptp_clock_index_cb) 8872 { 8873 if (knet_hw_tstamp_ptp_clock_index_cb != NULL) { 8874 return -1; 8875 } 8876 knet_hw_tstamp_ptp_clock_index_cb = hw_tstamp_ptp_clock_index_cb; 8877 return 0; 8878 } 8879 8880 int 8881 bkn_hw_tstamp_ptp_clock_index_cb_unregister(knet_hw_tstamp_ptp_clock_index_cb_f hw_tstamp_ptp_clock_index_cb) 8882 { 8883 if (hw_tstamp_ptp_clock_index_cb == NULL || 8884 knet_hw_tstamp_ptp_clock_index_cb != hw_tstamp_ptp_clock_index_cb) { 8885 return -1; 8886 } 8887 knet_hw_tstamp_ptp_clock_index_cb = NULL; 8888 return 0; 8889 } 8890 8891 int 8892 bkn_hw_tstamp_rx_time_upscale_cb_register(knet_hw_tstamp_rx_time_upscale_cb_f hw_tstamp_rx_time_upscale_cb) 8893 { 8894 if (knet_hw_tstamp_rx_time_upscale_cb != NULL) { 8895 return -1; 8896 } 8897 knet_hw_tstamp_rx_time_upscale_cb = hw_tstamp_rx_time_upscale_cb; 8898 return 0; 8899 } 8900 8901 int 8902 bkn_hw_tstamp_rx_time_upscale_cb_unregister(knet_hw_tstamp_rx_time_upscale_cb_f hw_tstamp_rx_time_upscale_cb) 8903 { 8904 if (hw_tstamp_rx_time_upscale_cb == NULL || 8905 knet_hw_tstamp_rx_time_upscale_cb != hw_tstamp_rx_time_upscale_cb) { 8906 return -1; 8907 } 8908 knet_hw_tstamp_rx_time_upscale_cb = NULL; 8909 return 0; 8910 } 8911 8912 LKM_EXPORT_SYM(bkn_rx_skb_cb_register); 8913 LKM_EXPORT_SYM(bkn_rx_skb_cb_unregister); 8914 LKM_EXPORT_SYM(bkn_tx_skb_cb_register); 8915 LKM_EXPORT_SYM(bkn_tx_skb_cb_unregister); 8916 LKM_EXPORT_SYM(bkn_filter_cb_register); 8917 LKM_EXPORT_SYM(bkn_filter_cb_unregister); 8918 LKM_EXPORT_SYM(bkn_hw_tstamp_enable_cb_register); 8919 LKM_EXPORT_SYM(bkn_hw_tstamp_enable_cb_unregister); 8920 LKM_EXPORT_SYM(bkn_hw_tstamp_disable_cb_register); 8921 LKM_EXPORT_SYM(bkn_hw_tstamp_disable_cb_unregister); 8922 LKM_EXPORT_SYM(bkn_hw_tstamp_tx_time_get_cb_register); 8923 LKM_EXPORT_SYM(bkn_hw_tstamp_tx_time_get_cb_unregister); 8924 LKM_EXPORT_SYM(bkn_hw_tstamp_tx_meta_get_cb_register); 8925 LKM_EXPORT_SYM(bkn_hw_tstamp_tx_meta_get_cb_unregister); 8926 LKM_EXPORT_SYM(bkn_hw_tstamp_ptp_clock_index_cb_register); 8927 LKM_EXPORT_SYM(bkn_hw_tstamp_ptp_clock_index_cb_unregister); 8928 LKM_EXPORT_SYM(bkn_hw_tstamp_rx_time_upscale_cb_register); 8929 LKM_EXPORT_SYM(bkn_hw_tstamp_rx_time_upscale_cb_unregister);