atp.c (147501B)
1 /* 2 * 3 * This license is set out in https://raw.githubusercontent.com/Broadcom-Network-Switching-Software/OpenBCM/master/Legal/LICENSE file. 4 * 5 * Copyright 2007-2019 Broadcom Inc. All rights reserved. 6 * 7 * File: atp.c 8 * Purpose: Acknowledged Transport Protocol and 9 * Best Effort Transport 10 * Requires: CPU-to-CPU module 11 * 12 * ATP is based on Jerry Zhao's implementation of "RDP", the reliable 13 * data protocol. ATP provides reliable packet transfer between 14 * applications on different CPUs. Reliability is achieved by 15 * sending an ACK for each packet exchanged. However, a packet 16 * may be segmented. ACKs may not (or may) be sent for each segment. 17 * 18 * When an ACK is not required, the transport is called BET for Best 19 * Effort Transport. 20 * 21 * The requirement of an ACK may be relaxed in several ways: 22 * 23 * 1. The client may be registered as a BET client. 24 * 2. An atp_tx request can be marked "no ACK required". 25 * This overrides the client's setting. 26 * 3. The CPU may be marked as "no ACK required" and all packets 27 * destined for that CPU will be marked no-ack. This is used 28 * in particular when ATP is used to communicate across the 29 * Linux user/kernel boundary. This overrides the client's 30 * setting. 31 * 32 * In any of these cases, a flag is set in the transmitted packet 33 * and this alone determines whether the receive side will send an 34 * ACK for the packet. 35 * 36 * On a per-CPU basis, the atp_tx operation can be short-circuited and 37 * an application's callback registered to replace the operation. This 38 * is done by calling atp_tx_override_set, giving the destination key 39 * and the override function. Setting this function to NULL will 40 * revert to normal atp_tx operation. 41 * 42 * When the atp function is overridden, the packet format must be 43 * carefully formed. If it matches the atp format, the receiving side 44 * may send the packet into the atp stack. This is okay, but the 45 * override TX function must abide by all ATP conventions (which are 46 * not that well documented.) Alternatively (and more likely) the 47 * application will handle the reception of the packet. It can then 48 * insert the packet into the callback portion of ATP by calling 49 * atp_rx_inject. The two (extra) pieces of information that must be 50 * provided to this call are the source CPU key and the client ID. 51 * 52 * Note that overriding atp_tx must be done carefully when used 53 * in conjunction with TKS discovery protocols. One possibility is 54 * to use different keys for a given CPU to select the transport to 55 * use. 56 * 57 * Other Important Notes 58 * 59 * For an RX client, if "REASSEM_BUF" is _not_ specified and a 60 * multiple segment ATP transfer occurs (so the segments are communicated 61 * up to the callback), this is indicated by the 62 * "payload" parameter of the client callback be NULL. In this case, 63 * the pkt->pkt_data blocks are set up with even pointers (0, 2, 4...) 64 * pointing to the CPU transport headers of each segment and 65 * the odd pointers (1, 3, 5...) pointing to the data portion of 66 * each segment. pkt->blk_count is 2 * (number of segments) in this 67 * case. The data pointers for each segment must be freed, but only 68 * one for each pair. That is, free either the transport header pointer 69 * or the data pointer, but not both. 70 * If only one segment occurs, then "payload" is set to point to 71 * the proper location in the single buffer. 72 * 73 * Table of contents (sections to look for below): 74 * 75 * TX thread 76 * Check for retransmit timeouts 77 * Check BET queue separately 78 * RX thread 79 * Check for RX callbacks that are ready 80 * Received Packet handling 81 * Handle ATP data (create/update RX transaction) 82 * Handle BET data (make callback from RX handler) 83 * Handle ACK (update TX operation) 84 * ACK out (sent immediately from RX handler) 85 * ATP API functions 86 * Config functions 87 * start/stop 88 * atp_register/unregister (clients) 89 * atp_tx 90 */ 91 92 #include <shared/bsl.h> 93 94 #include <assert.h> 95 96 #define ATP_ASSERT(stuff) assert stuff 97 98 #include <shared/idents.h> 99 100 #include <sal/core/time.h> 101 #include <sal/core/sync.h> 102 #include <sal/core/libc.h> 103 #include <sal/core/thread.h> 104 #include <shared/alloc.h> 105 106 #include <sal/appl/sal.h> /* sal_dma_alloc/_free */ 107 108 #include <bcm/types.h> 109 #include <bcm/pkt.h> 110 #include <bcm/rx.h> 111 #include <bcm/error.h> 112 113 #include <appl/cputrans/cpu2cpu.h> 114 #include <appl/cputrans/next_hop.h> 115 #include <appl/cputrans/atp.h> 116 #include <appl/cputrans/cputrans.h> 117 #include <appl/cpudb/cpudb.h> 118 119 #include "atp_int.h" 120 #include "t_util.h" 121 122 /**************************************************************** 123 * 124 * Client pointers 125 * A short array, hashed by client ID. 126 */ 127 128 #define _ATP_CLIENT_HASH(client_id) (client_id % _ATP_CLIENT_HASH_MAX) 129 _atp_client_t *_atp_client_buckets[_ATP_CLIENT_HASH_MAX]; 130 131 /* Include deleted clients */ 132 #define FOREACH_CLIENT(_client, _bkt) \ 133 for (_bkt = 0; _bkt < _ATP_CLIENT_HASH_MAX; _bkt++) \ 134 for (_client = _atp_client_buckets[_bkt]; _client != NULL; \ 135 _client = _client->next) 136 137 /* Use with ATP client flags */ 138 #define CLI_IS_BET(_cli) ((_cli)->flags & ATP_F_NO_ACK) 139 #define CLI_IS_NEXT_HOP(_cli) ((_cli)->flags & ATP_F_NEXT_HOP) 140 141 /* ATP keeps its own hash of CPUDB keys */ 142 143 /* 144 * Per-CPU flags: 145 * CPU_NO_ACK Force NO ACK on transmits to this CPU 146 * even for reliable calls. 147 */ 148 149 typedef struct _atp_cpu_info_s _atp_cpu_info_t; 150 struct _atp_cpu_info_s { 151 cpudb_key_t key; 152 uint32 flags; 153 atp_tx_f override_tx; /* Use override TX if !NULL */ 154 }; 155 #define _ATP_CPU_VALID 0x1 /* Is entry occupied */ 156 #define _ATP_CPU_NO_ACK 0x2 /* All TX to this CPU are no-ack */ 157 #define _ATP_TX_CXN_INIT 0x4 /* Has a reliable TX convo started */ 158 #define _ATP_RX_CXN_INIT 0x8 /* Has a reliable RX convo started */ 159 160 static int atp_cpu_max; 161 162 static _atp_cpu_info_t _atp_cpu_info[CPUDB_CPU_MAX]; 163 164 #define CPU_VALID_IDX(_idx) ((_idx) >= 0 && ((_idx) < CPUDB_CPU_MAX)) 165 166 #define CPU_VALID(_idx) (CPU_VALID_IDX(_idx) && \ 167 (_atp_cpu_info[_idx].flags & _ATP_CPU_VALID)) 168 169 #define CPU_KEY(_idx) \ 170 (*(CPU_VALID(_idx) ? &_atp_cpu_info[_idx].key : &cpudb_bcast_key)) 171 172 /* This tracks number of times DB has been updated; used for TX transmits */ 173 static int atp_db_update_count; 174 175 176 #if defined(BROADCOM_DEBUG) && defined(ATP_LONG_CALLBACK_TRACKER) 177 volatile sal_usecs_t atp_rx_in; 178 volatile sal_usecs_t atp_rx_out; 179 volatile int atp_rx_long_callbacks; 180 volatile atp_client_cb_f atp_rx_long_cb_ptr; 181 volatile atp_client_cb_f atp_rx_ptr; 182 #endif /* BROADCOM_DEBUG && ATP_LONG_CALLBACK_TRACKER */ 183 184 /**************************************************************** 185 * Local configuration variables 186 */ 187 static int atp_tx_thread_priority = ATP_THREAD_PRIORITY_DEFAULT; 188 static int atp_rx_thread_priority = ATP_THREAD_PRIORITY_DEFAULT; 189 190 /* 191 * The driver list of low level transport calls; currently only 192 * used for RX registration and init-time allocation. 193 */ 194 static bcm_trans_ptr_t *_atp_trans_ptr = &bcm_trans_ptr; 195 196 /**************************************************************** 197 * 198 * Threads and synchronization 199 * 200 * There are two threads in ATP: ATP_RX and ATP_TX. In addition, 201 * callbacks are registered with RX, so that thread is involved. 202 * Finally, application threads will call ATP functions. 203 * 204 * ATP_TX takes care of retransmitting packets (when ACKs do not 205 * arrive in time) and indicating timeouts. It also processes 206 * packets received from BCM-RX that have the ACK opcode. 207 * 208 * Most of ATP RX action happens in RX packet handling callbacks, 209 * _atp_rx_callback and _atp_next_hop_callback which are registered 210 * to get packets from lower layers. But ATP callbacks (to ATP 211 * client registered functions) happen in the ATP_RX thread. 212 * 213 * Synchronization, 11/05: Two mutexes, one for RX and one for 214 * TX, are now used. To change the configuration, both should 215 * be taken. RX callbacks hold the RX mutex and may call TX, 216 * so the order of taking mutexes MUST be: 217 * 218 * ATP_RX_LOCK -> ATP_TX_LOCK -> ATP_TX_UNLOCK -> ATP_RX_UNLOCK 219 * 220 * In addition (to allow TX loopback to enqueue an RX transaction) 221 * there is a low level queue mutex used on RX transaction queues. 222 * 223 * Programming requirement: 224 * An ATP_TX callback MUST NOT call an ATP function that takes 225 * the RX lock. 226 */ 227 228 static sal_mutex_t atp_tx_mutex; /* High level TX thread */ 229 static sal_mutex_t atp_rx_mutex; /* High level RX thread */ 230 static sal_mutex_t atp_rxq_mutex; /* Low level queue protection */ 231 232 static sal_sem_t atp_tx_sem; 233 static sal_sem_t atp_rx_sem; 234 235 static volatile sal_thread_t atp_tx_thread_id = SAL_THREAD_ERROR; 236 static volatile sal_thread_t atp_rx_thread_id = SAL_THREAD_ERROR; 237 static volatile int atp_tx_thread_exit; /* Forces exit when true */ 238 static volatile int atp_rx_thread_exit; /* Forces exit when true */ 239 240 /* Transmit thread lock */ 241 #define ATP_TX_LOCK sal_mutex_take(atp_tx_mutex, sal_sem_FOREVER) 242 #define ATP_TX_UNLOCK sal_mutex_give(atp_tx_mutex) 243 244 /* Receive thread lock */ 245 #define ATP_RX_LOCK sal_mutex_take(atp_rx_mutex, sal_sem_FOREVER) 246 #define ATP_RX_UNLOCK sal_mutex_give(atp_rx_mutex) 247 248 /* Low level RX transaction queue lock */ 249 #define ATP_RXQ_LOCK sal_mutex_take(atp_rxq_mutex, sal_sem_FOREVER) 250 #define ATP_RXQ_UNLOCK sal_mutex_give(atp_rxq_mutex) 251 252 /* Configuration (both TX and RX) lock */ 253 #define ATP_LOCK do { ATP_RX_LOCK; ATP_TX_LOCK; } while (0) 254 #define ATP_UNLOCK do { ATP_TX_UNLOCK; ATP_RX_UNLOCK; } while (0) 255 256 static volatile int init_done; 257 static volatile int base_init_done; 258 static volatile int _atp_running; /* Busy? */ 259 260 /**************************************************************** 261 * Preallocation pointers 262 */ 263 264 static int atp_retry_timeout = ATP_RETRY_TIMEOUT_DEFAULT; 265 static int atp_retry_count = ATP_RETRY_COUNT_DEFAULT; 266 static int atp_tx_pool_size = ATP_TX_TRANSACT_DEFAULT; 267 static int atp_rx_pool_size = ATP_RX_TRANSACT_DEFAULT; 268 static atp_timeout_cb_f atp_timeout_cb; 269 270 /* For now, one BET queue; maybe by priority later */ 271 static _atp_tx_trans_t *bet_queue; 272 static _atp_tx_trans_t *bet_queue_tail; 273 274 /**************************************************************** 275 * Parameters 276 */ 277 278 static uint32 _atp_flags; /* Setup flags; see atp.h */ 279 static uint32 _atp_units; /* Unit bitmap for register/unregister */ 280 281 static volatile int _atp_seg_len = ATP_SEG_LEN_DEFAULT; 282 283 /**************************************************************** 284 * Default client parameters 285 */ 286 287 static int _atp_cos = ATP_COS_DEFAULT; 288 static int _atp_vlan = ATP_VLAN_DEFAULT; 289 290 291 /**************************************************************** 292 * Counters 293 */ 294 295 static volatile int bet_rx_drop; /* Number of BET RX dropped pkts */ 296 static volatile int atp_rx_drop; /* Number of ATP RX dropped pkts */ 297 static volatile int mem_rx_drop; /* Could alloc mem and dropped pkt */ 298 static volatile int slf_rx_drop; /* Source look up failure */ 299 300 static volatile int old_rx_trans_drop; /* Source look up failure */ 301 302 static volatile int tx_sleep_count; /* Current number of TX waits */ 303 304 #if defined(BROADCOM_DEBUG) 305 #define INCR_COUNTER(counter) ++(counter) 306 static volatile uint32 rxt_create; /* RX trans create */ 307 static volatile uint32 txt_create; 308 static volatile uint32 rxt_free; /* RX trans free */ 309 static volatile uint32 rxraw_free; /* RX trans free, raw */ 310 static volatile uint32 rxraw_grab; /* RX trans grab, raw */ 311 static volatile uint32 txraw_grab; 312 static volatile uint32 cli_del_tx_busy; /* Client delete failed for busy TX */ 313 static volatile uint32 clients_deleted; 314 static volatile uint32 reassem_alloc_fail; 315 static volatile uint32 rx_trans_fail; 316 static volatile uint32 rxt_pkt_alloc_fail; 317 static volatile uint32 rx_mseg_alloc_fail; 318 static volatile uint32 tx_trans_fail; 319 static volatile uint32 txt_pkt_alloc_fail; 320 static volatile uint32 lb_buf_alloc_fail; 321 static volatile uint32 gc_deferrals; 322 static volatile uint32 gc_blocked; 323 static volatile uint32 tx_timeout_cnt; 324 static volatile uint32 tx_retry_cnt; 325 static volatile uint32 stale_rx_trans; 326 static volatile uint32 rx_pkt_drops; 327 static volatile uint32 ack_pkt_drops; 328 static volatile uint32 tx_data_alloc_fail; 329 static volatile uint32 rx_data_alloc_fail; 330 static volatile uint32 invalid_client_cnt; 331 static volatile uint32 invalid_dest_cpu_cnt; 332 static volatile uint32 lb_pkt_send_fail; 333 static volatile uint32 tx_simple_send_fail; 334 static volatile uint32 tx_trans_setup_fail; 335 static volatile uint32 tx_send_fail; 336 static volatile uint32 atp_not_running; 337 #else 338 #define INCR_COUNTER(counter) 339 #endif /* BROADCOM_DEBUG */ 340 341 /* 342 * Both ATP-TX and ATP-RX need to receive packets. Each has its 343 * own queue, both protected by the following mutex. 344 * If both this and ATP_LOCK both are taken, the order should be 345 * ATP_LOCK -> ATP_PKT_DATA_LOCK -> ATP_PKT_DATA_UNLOCK -> ATP_UNLOCK 346 */ 347 348 static sal_mutex_t atp_pkt_data_mutex; 349 350 #define ATP_PKT_DATA_LOCK sal_mutex_take(atp_pkt_data_mutex, \ 351 sal_sem_FOREVER) 352 #define ATP_PKT_DATA_UNLOCK sal_mutex_give(atp_pkt_data_mutex) 353 354 _atp_pkt_data_t *_atp_pkt_data_freelist; 355 356 _atp_pkt_data_t *_atp_rcv_data_queue; /* For ATP-RX */ 357 _atp_pkt_data_t *_atp_rcv_data_queue_tail; 358 359 _atp_pkt_data_t *_atp_trx_data_queue; /* For ATP-TX */ 360 _atp_pkt_data_t *_atp_trx_data_queue_tail; 361 362 /**************************************************************** 363 * 364 * TX and RX Transactions: 365 * 366 * Each ATP operation results in a TX transaction being tracked 367 * by the sender and an RX transaction being tracked by the 368 * receiver. Transactions are maintained on a per-client, 369 * per-CPU (other side) basis. 370 * 371 * Whether a transaction has been sent to/received from a 372 * CPU is indicated by the TX_CXN/RX_CXN bit in atp_cpu_info. 373 * These are detected by sequence numbers as described below. 374 * 375 * tx/rx_trans_pool is an allocation pointer used for freeing later. 376 * tx/rx_trans_freelist is a pointer to the head of the freelist 377 * 378 * The queued transactions are kept in the client structure, 379 * indexed by CPU. 380 * 381 * RX transactions have an ACK packet data pointer associated 382 * at allocation time which is reused. They use one packet which 383 * holds pointers to all the data that arrives. Each data packet 384 * uses 2 pointer blocks, one for the header and one for the 385 * start of the payload. 386 * 387 * For the situation where the queue is empty, but an old 388 * transaction is received, an ACK packet data buffer is kept 389 * in the client structure. 390 * 391 * ATP transactions are tracked by "sequence numbers" which are 392 * kept on a per-client, per-CPU basis and generated on TX. 393 * The ATP sequence number 0 is treated specially. It is only 394 * used on TX on the first transaction from a CPU (no matter what 395 * client). This marks the beginning of a "connection" which 396 * is a series of unidirectional transactions between two CPUs. 397 * If the sequence number 0 is ever seen again, this is an 398 * indication that the remote side has reset and the transaction 399 * information for that CPU is purged. 400 * 401 * Otherwise, sequence numbers are tracked per (client, dest-CPU) 402 * pair. They are set to 1 on a "wrap" condition. 403 */ 404 405 /* Transmit transaction queues, preallocated */ 406 static _atp_tx_trans_t *tx_trans_pool; /* Allocated */ 407 static _atp_tx_trans_t *tx_trans_freelist; /* Current free list */ 408 409 static _atp_rx_trans_t *rx_trans_pool; /* Allocated once */ 410 static _atp_rx_trans_t *rx_trans_freelist; /* Current free list */ 411 412 static uint8 *ack_pkt_data; 413 int atp_ack_pkt_data_from_heap = 0; /* DMA pool to begin-with */ 414 415 /* How many TX operations pending */ 416 volatile int atp_tx_pending; 417 418 static cpudb_key_t _atp_local_key; 419 420 #define IS_LOCAL_CPU_KEY(_key) (CPUDB_KEY_EQUAL(_atp_local_key, _key)) 421 422 #define ATP_TX_THREAD_WAKE sal_sem_give(atp_tx_sem) 423 #define ATP_RX_THREAD_WAKE sal_sem_give(atp_rx_sem) 424 425 /* If init not done, call init function */ 426 #define ATP_INIT_CHECK \ 427 if (!init_done) BCM_IF_ERROR_RETURN(_atp_init()) 428 429 /* If init not done, call init function */ 430 #define BASE_INIT_CHECK \ 431 if (!base_init_done) BCM_IF_ERROR_RETURN(_atp_base_init()) 432 433 #define ATP_TIMEOUT (atp_retry_timeout * atp_retry_count) 434 435 /* Don't need to timeout if nothing happening */ 436 #define ATP_LONG_TIMEOUT sal_sem_FOREVER 437 438 /* 439 * Set up a packet block entry from the given data; the CPUTRANS header 440 * goes in the first block and the payload ptr in the second. 441 */ 442 #define SET_PKT_BLK_DATA(_seg, _pkt, _buf, _len) \ 443 do { \ 444 int _d_ofs; \ 445 _d_ofs = 2 * (_seg); \ 446 /* CPUTRANS header.... */ \ 447 (_pkt)->pkt_data[_d_ofs].data = _buf; \ 448 (_pkt)->pkt_data[_d_ofs++].len = CPUTRANS_HEADER_BYTES; \ 449 /* .... Payload data */ \ 450 (_pkt)->pkt_data[_d_ofs].data = (_buf) + CPUTRANS_HEADER_BYTES; \ 451 (_pkt)->pkt_data[_d_ofs].len = \ 452 (_len) - CPUTRANS_HEADER_BYTES; \ 453 } while (0) 454 455 456 /* 457 * Get the current sequence number for a BET transaction controlled by pkt 458 * We only use this cookie while the packet is being accumulated. 459 */ 460 461 #define BET_RX_SEQ_NUM(pkt) (PTR_TO_INT((pkt)->cookie)) 462 #define BET_RX_SEQ_NUM_SET(pkt, seq) (pkt)->cookie = INT_TO_PTR(((int)(seq))) 463 464 /* 465 * Set and update a sequence number; on wrap, skip 0 which indicates a reset. 466 * This should never return 0 which is a special marker for start of convo. 467 */ 468 #define _SEQ_NUM_GET(_dest, _sn) \ 469 if (++(_sn) == 0) (_dest) = (_sn) = 1; \ 470 else (_dest) = (_sn) 471 472 /* 473 * Set a TX sequence number; depends on if BET or ACK 474 * If BET, _cpu is ignored. 475 * If reliable, check CPU's TX_CXN_INIT flag and use SN 0 if not set. 476 * Set the TX_CXN_INIT bit at that time. 477 */ 478 #define TX_SEQ_NUM_GET(no_ack, _dest, _cli, _cpu) do { \ 479 if (no_ack) { \ 480 _SEQ_NUM_GET(_dest, (_cli)->bet_tx_seq_num); \ 481 } else { \ 482 if (!(_atp_cpu_info[_cpu].flags & _ATP_TX_CXN_INIT)) { \ 483 _dest = (_cli)->cpu[_cpu].tx_seq_num = 0; \ 484 _atp_cpu_info[_cpu].flags |= _ATP_TX_CXN_INIT; \ 485 } else { \ 486 _SEQ_NUM_GET(_dest, (_cli)->cpu[_cpu].tx_seq_num); \ 487 } \ 488 } \ 489 } while (0) 490 491 #define TX_BET_SEQ_NUM_GET(_dest, _cli) \ 492 _SEQ_NUM_GET(_dest, (_cli)->bet_tx_seq_num) 493 494 STATIC void atp_tx_thread(void *cookie); 495 STATIC void atp_rx_thread(void *cookie); 496 STATIC bcm_rx_t _atp_rx_callback(int unit, bcm_pkt_t *pkt, void *cookie); 497 498 STATIC int _atp_base_init(void); 499 STATIC int _atp_init(void); 500 STATIC void atp_cleanup(void); 501 502 STATIC bcm_rx_t _atp_next_hop_callback(cpudb_key_t src_key, 503 int mplx_num, 504 int unit, 505 int port, 506 uint8* pkt_buf, 507 int len, 508 void *cookie); 509 510 STATIC int _atp_end_threads(int retries); 511 512 STATIC void bet_nh_free_tx_cb(int rv, uint8 *pkt_buf, void *cookie); 513 STATIC void bet_c2c_free_tx_cb(uint8 *pkt_buf, void *cookie); 514 515 STATIC void _atp_c2c_tx_callback(uint8 *pkt_buf, void *cookie); 516 STATIC void _atp_nh_tx_callback(int rv, uint8 *pkt_buf, void *cookie); 517 518 STATIC void atp_rx_ack(_atp_client_t *cli, int cpu, _atp_rx_trans_t *rx_trans, 519 uint16 seq_num, 520 uint8 *pkt_buf, int pkt_len); 521 522 STATIC void bet_rx_pkt_drop(_atp_client_t *client, int src_cpu, int err); 523 524 STATIC int atp_db_update_locked(cpudb_ref_t db_ref); 525 526 STATIC bcm_rx_t atp_data_handle(int src_cpu, uint8 *pkt_buf, int pkt_len, 527 _atp_hdr_t *_atp_hdr); 528 STATIC bcm_rx_t bet_data_handle(int src_cpu, uint8 *pkt_buf, int len, 529 _atp_hdr_t *_atp_hdr); 530 STATIC bcm_rx_t atp_ack_handle(int src_cpu, _atp_hdr_t *_atp_hdr); 531 532 /* 533 * Function: 534 * _atp_hdr_pack/unpack 535 * Purpose: 536 * Pack/Unpack and ATP header into/from a packet buffer 537 * Parameters: 538 * 539 * Returns: 540 * BCM_E_XXX 541 */ 542 543 /* Set the sequence number given the start of CPU trans header */ 544 #define ATP_SEQ_NUM_SET(hdr_data, val) \ 545 PACK_SHORT(&(hdr_data[ATP_SEQ_NUM_OFS]), val) 546 547 /* Header offset; client ID is packed as a uint16 */ 548 #define _ATP_HDR_OPCODE_OFFSET ((sizeof(uint16) * 5) + sizeof(uint32) + 2) 549 550 STATIC INLINE uint8 _atp_opcode_get(uint8 *hdr_data) 551 { 552 hdr_data = ATP_HEADER_START(hdr_data); 553 return hdr_data[_ATP_HDR_OPCODE_OFFSET]; 554 } 555 556 STATIC INLINE void 557 _atp_hdr_unpack(uint8 *hdr_data, _atp_hdr_t *hdr) 558 { 559 uint8 *ptr; 560 uint16 val16; 561 562 ptr = ATP_HEADER_START(hdr_data); 563 UNPACK_SHORT(ptr, hdr->version); /* ATP version */ 564 ptr += sizeof(uint16); 565 UNPACK_SHORT(ptr, val16); /* clientid */ 566 hdr->client_id = val16; 567 ptr += sizeof(uint16); 568 UNPACK_LONG(ptr, hdr->hdr_flags); /* flags */ 569 ptr += sizeof(uint32); 570 UNPACK_SHORT(ptr, hdr->seq_num); /* Sequence number */ 571 ptr += sizeof(uint16); 572 UNPACK_SHORT(ptr, hdr->tot_bytes); /* total payload length */ 573 ptr += sizeof(uint16); 574 UNPACK_SHORT(ptr, hdr->start_byte); /* payload offset */ 575 ptr += sizeof(uint16); 576 hdr->tot_segs = *ptr++; /* total number of segments */ 577 hdr->segment = *ptr++; /* segment */ 578 hdr->opcode = *ptr++; /* opcode */ 579 hdr->cos = *ptr++; /* cos */ 580 } 581 582 STATIC INLINE void 583 _atp_hdr_pack(uint8 *hdr_data, _atp_hdr_t *hdr) 584 { 585 uint8 *ptr; 586 uint16 val16; 587 588 ptr = ATP_HEADER_START(hdr_data); 589 PACK_SHORT(ptr, ATP_VERSION); /* ATP version */ 590 ptr += sizeof(uint16); 591 val16 = hdr->client_id; 592 PACK_SHORT(ptr, val16); /* clientid */ 593 ptr += sizeof(uint16); 594 PACK_LONG(ptr, hdr->hdr_flags); /* flags */ 595 ptr += sizeof(uint32); 596 PACK_SHORT(ptr, hdr->seq_num); /* Sequence number */ 597 ptr += sizeof(uint16); 598 PACK_SHORT(ptr, hdr->tot_bytes); /* Total payload length */ 599 ptr += sizeof(uint16); 600 PACK_SHORT(ptr, hdr->start_byte); /* Offset in payload */ 601 ptr += sizeof(uint16); 602 *ptr++ = hdr->tot_segs; /* Total number of segments */ 603 *ptr++ = hdr->segment; /* Which segment */ 604 *ptr++ = hdr->opcode; /* opcode */ 605 *ptr++ = hdr->cos; /* cos */ 606 } 607 608 609 /***************************************************************** 610 * 611 * Client and CPU info management 612 * Client allocation/find/free routines 613 * The clients are organized in a small hash, each bucket 614 * being a linked list. 615 */ 616 617 /* 618 * Return a pointer to the client structure for the ID, or NULL if not found 619 * Assumes lock is held. 620 */ 621 622 STATIC _atp_client_t * 623 client_find(int client_id) 624 { 625 int b_idx; /* Bucket index */ 626 _atp_client_t *cli; 627 628 b_idx = _ATP_CLIENT_HASH(client_id); 629 cli = _atp_client_buckets[b_idx]; 630 while (cli != NULL) { 631 /* Ignore "deleted" clients */ 632 if (cli->client_id == client_id) { 633 break; 634 } 635 cli = cli->next; 636 } 637 638 return cli; 639 } 640 641 #define RELEASE_CLIENT_DATA(cli) \ 642 _atp_trans_ptr->tp_data_free(_atp_trans_ptr->tp_unit, \ 643 (cli)->cpu[0].rx_ack_data) 644 645 /* Assumes ATP_LOCK held */ 646 STATIC _atp_client_t * 647 client_id_add(int client_id) 648 { 649 int b_idx; /* Bucket index */ 650 _atp_client_t *new_ptr = NULL; 651 int cpu; 652 uint8 *rx_ack_ptr; 653 int bytes; 654 655 new_ptr = sal_alloc(sizeof(_atp_client_t), "ATP-cli"); 656 if (new_ptr == NULL) { 657 return NULL; 658 } 659 sal_memset((void *)new_ptr, 0, sizeof(_atp_client_t)); 660 661 /* Allocate RX ack data, per-client, per-CPU */ 662 bytes = _ATP_ACK_BYTES * CPUDB_CPU_MAX; 663 rx_ack_ptr = NULL; 664 _atp_trans_ptr->tp_data_alloc(_atp_trans_ptr->tp_unit, 665 bytes, 0, (void*)&rx_ack_ptr); 666 if (rx_ack_ptr == NULL) { 667 sal_free((void *)new_ptr); 668 return NULL; 669 } 670 for (cpu = 0; cpu < CPUDB_CPU_MAX; cpu++) { 671 new_ptr->cpu[cpu].rx_ack_data = &rx_ack_ptr[cpu * _ATP_ACK_BYTES]; 672 } 673 674 new_ptr->client_id = client_id; 675 676 /* Set up default client values */ 677 new_ptr->cos = _atp_cos; 678 new_ptr->vlan = _atp_vlan; 679 680 /* Link client into list */ 681 b_idx = _ATP_CLIENT_HASH(client_id); 682 new_ptr->next = _atp_client_buckets[b_idx]; 683 _atp_client_buckets[b_idx] = new_ptr; 684 685 return new_ptr; 686 } 687 688 /* Add a key to db; assumes key is not already present */ 689 690 STATIC int 691 _atp_key_add(cpudb_key_t key) 692 { 693 int idx; 694 695 for (idx = 0; idx < CPUDB_CPU_MAX; idx++) { 696 if (!(_atp_cpu_info[idx].flags & _ATP_CPU_VALID)) { 697 sal_memset(&_atp_cpu_info[idx], 0, sizeof(_atp_cpu_info_t)); 698 _atp_cpu_info[idx].flags = _ATP_CPU_VALID; 699 CPUDB_KEY_COPY(_atp_cpu_info[idx].key, key); 700 LOG_VERBOSE(BSL_LS_TKS_ATP, 701 (BSL_META("ATP: Adding CPU %d " CPUDB_KEY_FMT_EOLN), 702 idx, 703 CPUDB_KEY_DISP(key))); 704 if (idx >= atp_cpu_max) { 705 atp_cpu_max = idx+1; 706 } 707 return idx; 708 } 709 } 710 711 return -1; 712 } 713 714 /* Find a key in the DB */ 715 716 STATIC int 717 _atp_key_lookup(cpudb_key_t key) 718 { 719 int idx; 720 721 for (idx = 0; idx < CPUDB_CPU_MAX; idx++) { 722 if ((_atp_cpu_info[idx].flags & _ATP_CPU_VALID) && 723 CPUDB_KEY_EQUAL(_atp_cpu_info[idx].key, key)) { 724 return idx; 725 } 726 } 727 728 return -1; 729 } 730 731 /**************************************************************** 732 * 733 * Low level data handling 734 */ 735 736 /* 737 * Check if BET data is ready for service. This is the case when all 738 * the data pointers are non-zero. 739 */ 740 741 STATIC int 742 packet_data_done(bcm_pkt_t *pkt) 743 { 744 int i; 745 746 for (i = 0; i < pkt->blk_count; i += 2) { 747 if (pkt->pkt_data[i].data == NULL) { 748 return FALSE; 749 } 750 } 751 752 return TRUE; 753 } 754 755 /**************************************************************** 756 * 757 * RX handling 758 * 759 * TX and RX are mostly independent. The exception is that 760 * ACKs for TX are received by RX. 761 */ 762 763 /* 764 * If extra data was allocated, it is pointed to by pkt->alloc_ptr 765 */ 766 767 STATIC void 768 _atp_rx_pkt_free(bcm_pkt_t *pkt) 769 { 770 int i; 771 772 if (pkt == NULL) { 773 return; 774 } 775 776 /* Free the alloc ptr if present */ 777 if (pkt->alloc_ptr != NULL) { 778 _atp_trans_ptr->tp_data_free(_atp_trans_ptr->tp_unit, pkt->alloc_ptr); 779 pkt->alloc_ptr = NULL; 780 } 781 782 /* Free the data pointers if present */ 783 for (i = 0; i < pkt->blk_count; i += 2) { 784 if (pkt->pkt_data[i].data != NULL) { 785 _atp_trans_ptr->tp_data_free(_atp_trans_ptr->tp_unit, 786 pkt->pkt_data[i].data); 787 pkt->pkt_data[i].data = NULL; 788 } 789 } 790 791 /* Return the packet to the CPU transport pool */ 792 cputrans_rx_pkt_free(pkt); 793 } 794 795 /* 796 * Try to delete a transaction; if it's sync and sem has not 797 * been given, give the sem and return. 798 */ 799 800 STATIC void 801 atp_tx_trans_delete(_atp_tx_trans_t *tx_trans) 802 { 803 _atp_client_t *client; /* Controlling client */ 804 int cpu; 805 806 client = tx_trans->client; 807 cpu = tx_trans->dest_cpu; 808 809 if (tx_trans->flags & _ATP_TX_F_ENQUEUED) { /* dequeue */ 810 if (tx_trans->prev != NULL) { 811 tx_trans->prev->next = tx_trans->next; 812 } else { /* First on list */ 813 if (TX_TRANS_NO_ACK(tx_trans)) { 814 bet_queue = tx_trans->next; 815 } else { 816 if (CPU_VALID_IDX(cpu)) { 817 client->cpu[cpu].tx_trans = tx_trans->next; 818 } else { 819 LOG_VERBOSE(BSL_LS_TKS_ATP, 820 (BSL_META("ATP: invalid cpu index %d\n"), 821 cpu)); 822 } 823 } 824 } 825 826 if (tx_trans->next != NULL) { 827 tx_trans->next->prev = tx_trans->prev; 828 } else { /* Last on list */ 829 if (TX_TRANS_NO_ACK(tx_trans)) { 830 bet_queue_tail = tx_trans->prev; 831 } else { 832 if (CPU_VALID_IDX(cpu)) { 833 client->cpu[cpu].tx_tail = tx_trans->prev; 834 } else { 835 LOG_VERBOSE(BSL_LS_TKS_ATP, 836 (BSL_META("ATP: invalid cpu index %d\n"), 837 cpu)); 838 } 839 } 840 } 841 842 if (TX_TRANS_ACK(tx_trans)) { 843 --atp_tx_pending; 844 } 845 846 tx_trans->flags &= ~_ATP_TX_F_ENQUEUED; 847 } 848 849 if ((tx_trans->tx_sem != NULL) && 850 (tx_trans->flags & _ATP_TX_F_SEM_WAITING)) { 851 LOG_DEBUG(BSL_LS_TKS_ATP, 852 (BSL_META("TT delete deferred %p\n"), 853 tx_trans)); 854 tx_trans->flags &= ~_ATP_TX_F_SEM_WAITING; 855 sal_sem_give(tx_trans->tx_sem); 856 return; 857 } 858 LOG_DEBUG(BSL_LS_TKS_ATP, 859 (BSL_META("TT delete %p\n"), 860 tx_trans)); 861 862 /* Deallocate the packet */ 863 if (tx_trans->pkt_list != NULL) { 864 ATP_ASSERT((tx_trans->pkt_list->next != tx_trans->pkt_list)); 865 cputrans_tx_pkt_list_free(tx_trans->pkt_list); 866 tx_trans->pkt_list = NULL; 867 } 868 869 if (tx_trans->tx_sem != NULL) { 870 sal_sem_destroy(tx_trans->tx_sem); 871 tx_trans->tx_sem = NULL; 872 } 873 tx_trans->next = tx_trans_freelist; 874 tx_trans_freelist = tx_trans; 875 } 876 877 STATIC void 878 atp_tx_trans_delete_all(_atp_client_t *client, int cpu) 879 { 880 _atp_tx_trans_t *tx_trans; 881 _atp_tx_trans_t *tx_trans_next; 882 883 ATP_TX_LOCK; 884 tx_trans = client->cpu[cpu].tx_trans; 885 while (tx_trans) { 886 887 /* Call TX callback if necessary */ 888 if (tx_trans->callback != NULL) { 889 tx_trans->callback(tx_trans->pkt_buf, tx_trans->cookie, 890 BCM_E_FAIL); 891 } 892 893 tx_trans_next = (_atp_tx_trans_t *)tx_trans->next; 894 atp_tx_trans_delete(tx_trans); 895 tx_trans = tx_trans_next; 896 } 897 ATP_TX_UNLOCK; 898 } 899 900 STATIC void 901 atp_rx_trans_enqueue(_atp_rx_trans_t *new_trans, _atp_client_t *client, 902 int cpu) 903 { 904 new_trans->next = NULL; 905 ATP_RXQ_LOCK; 906 if (client->cpu[cpu].rx_tail == NULL) { /* Queue now empty */ 907 client->cpu[cpu].rx_trans = client->cpu[cpu].rx_tail = new_trans; 908 } else { 909 new_trans->prev = client->cpu[cpu].rx_tail; 910 client->cpu[cpu].rx_tail->next = new_trans; 911 client->cpu[cpu].rx_tail = new_trans; 912 } 913 new_trans->flags |= _ATP_RX_F_ENQUEUED; 914 ATP_RXQ_UNLOCK; 915 } 916 917 918 /* Free and dequeue an RX transaction */ 919 920 STATIC void 921 atp_rx_trans_delete(_atp_rx_trans_t *rx_trans) 922 { 923 _atp_client_t *client; /* Controlling client */ 924 int cpu; 925 926 client = rx_trans->client; 927 cpu = rx_trans->src_cpu; 928 929 ATP_RXQ_LOCK; 930 if (rx_trans->flags & _ATP_RX_F_ENQUEUED) { /* dequeue */ 931 if (rx_trans->prev != NULL) { 932 rx_trans->prev->next = rx_trans->next; 933 } else { /* First on list */ 934 client->cpu[cpu].rx_trans = rx_trans->next; 935 } 936 937 if (rx_trans->next != NULL) { 938 rx_trans->next->prev = rx_trans->prev; 939 } else { /* Last on list */ 940 client->cpu[cpu].rx_tail = rx_trans->prev; 941 } 942 943 rx_trans->flags &= ~_ATP_RX_F_ENQUEUED; 944 } 945 ATP_RXQ_UNLOCK; 946 947 if (rx_trans->pkt != NULL) { 948 _atp_rx_pkt_free(rx_trans->pkt); 949 rx_trans->pkt = NULL; 950 } 951 952 if (rx_trans->lb_data != NULL) { 953 _atp_trans_ptr->tp_data_free(_atp_trans_ptr->tp_unit, 954 rx_trans->lb_data); 955 rx_trans->lb_data = NULL; 956 } 957 958 ATP_RXQ_LOCK; 959 rx_trans->next = rx_trans_freelist; 960 rx_trans_freelist = rx_trans; 961 ATP_RXQ_UNLOCK; 962 963 INCR_COUNTER(rxt_free); 964 LOG_DEBUG(BSL_LS_TKS_ATP, 965 (BSL_META("RT delete %p\n"), 966 rx_trans)); 967 } 968 969 STATIC void 970 atp_rx_trans_delete_all(_atp_client_t *client, int cpu) 971 { 972 _atp_rx_trans_t *rx_trans; 973 _atp_rx_trans_t *rx_trans_next; 974 975 rx_trans = client->cpu[cpu].rx_trans; 976 while (rx_trans) { 977 rx_trans_next = (_atp_rx_trans_t *)rx_trans->next; 978 atp_rx_trans_delete(rx_trans); 979 rx_trans = rx_trans_next; 980 } 981 } 982 983 STATIC void 984 _atp_cpu_purge(int idx) 985 { 986 int bkt; 987 _atp_client_t *client; 988 989 FOREACH_CLIENT(client, bkt) { 990 if (client->cpu[idx].cpu_flags & ATP_CPU_ACK_PENDING) { 991 LOG_WARN(BSL_LS_TKS_ATP, 992 (BSL_META("ATP: purge cpu %d while ack pending\n"), 993 idx)); 994 } 995 client->cpu[idx].cpu_flags = 0; 996 client->cpu[idx].tx_seq_num = 0; 997 client->cpu[idx].rx_seq_num = 0; 998 999 atp_tx_trans_delete_all(client, idx); 1000 if (client->cpu[idx].rx_tail != NULL) { 1001 LOG_WARN(BSL_LS_TKS_ATP, 1002 (BSL_META("ATP: purge cpu %d while RX trans pending\n"), 1003 idx)); 1004 } 1005 atp_rx_trans_delete_all(client, idx); 1006 if (client->cpu[idx].bet_rx_pkts != NULL) { 1007 bet_rx_pkt_drop(client, idx, FALSE); 1008 } 1009 } 1010 1011 /* Clear convo init flags */ 1012 _atp_cpu_info[idx].flags &= ~(_ATP_TX_CXN_INIT | _ATP_RX_CXN_INIT); 1013 } 1014 1015 /* Assumes lock held */ 1016 1017 STATIC void 1018 _atp_cpu_remove(int idx) 1019 { 1020 LOG_VERBOSE(BSL_LS_TKS_ATP, 1021 (BSL_META("ATP: Removing CPU %d\n"), 1022 idx)); 1023 _atp_cpu_purge(idx); 1024 _atp_cpu_info[idx].flags = 0; 1025 } 1026 1027 /* 1028 * Delete a client. 1029 * If check_tx is set, then the TX queue is checked to see if any 1030 * transactions are _present_. If so, _E_BUSY is returned. 1031 * 1032 * Otherwise RX queue is emptied and the client is removed. 1033 */ 1034 1035 STATIC int 1036 client_delete(_atp_client_t *client, int check_tx) 1037 { 1038 int cpu; 1039 1040 if (check_tx) { 1041 /* Return BUSY if any TX transactions are present */ 1042 for (cpu = 0; cpu < CPUDB_CPU_MAX; cpu++) { 1043 if (client->cpu[cpu].tx_trans != NULL) { 1044 INCR_COUNTER(cli_del_tx_busy); 1045 return BCM_E_BUSY; 1046 } 1047 } 1048 } 1049 1050 INCR_COUNTER(clients_deleted); 1051 if (client->prev == NULL) { 1052 _atp_client_buckets[_ATP_CLIENT_HASH(client->client_id)] = client->next; 1053 } else { 1054 client->prev->next = client->next; 1055 } 1056 if (client->next != NULL) { 1057 client->next->prev = client->prev; 1058 } 1059 1060 /* Clean up old RX transactions */ 1061 for (cpu = 0; cpu < CPUDB_CPU_MAX; cpu++) { 1062 atp_rx_trans_delete_all(client, cpu); 1063 } 1064 1065 /* Clean up TX transactions */ 1066 for (cpu = 0; cpu < CPUDB_CPU_MAX; cpu++) { 1067 atp_tx_trans_delete_all(client, cpu); 1068 } 1069 1070 RELEASE_CLIENT_DATA(client); 1071 sal_free((void *)client); 1072 1073 return BCM_E_NONE; 1074 } 1075 1076 /* 1077 * Allocate a data buffer and copy the rx_trans data into it. 1078 * This reassembles the packet data into a uniform buffer. 1079 */ 1080 1081 STATIC uint8 * 1082 _pkt_reassem(_atp_rx_trans_t *rx_trans) 1083 { 1084 uint8 *pkt_buf; 1085 bcm_pkt_blk_t *blk; 1086 int i; 1087 int offset; 1088 1089 /* Allocate enough space for one CPUTRANS header all pkt data */ 1090 pkt_buf = NULL; 1091 _atp_trans_ptr->tp_data_alloc(_atp_trans_ptr->tp_unit, 1092 rx_trans->payload_len + CPUTRANS_HEADER_BYTES, 0, (void*)&pkt_buf); 1093 if (pkt_buf == NULL) { 1094 INCR_COUNTER(reassem_alloc_fail); 1095 return NULL; 1096 } 1097 1098 /* Copy in the first CPU header block */ 1099 blk = &rx_trans->pkt->pkt_data[0]; 1100 sal_memcpy(pkt_buf, blk->data, CPUTRANS_HEADER_BYTES); 1101 offset = CPUTRANS_HEADER_BYTES; 1102 1103 /* Copy in data from odd blocks of packet */ 1104 for (i = 0; i < rx_trans->_atp_hdr.tot_segs; i++) { 1105 blk = &rx_trans->pkt->pkt_data[2 * i + 1]; 1106 sal_memcpy(&pkt_buf[offset], blk->data, blk->len); 1107 offset += blk->len; 1108 } 1109 1110 return pkt_buf; 1111 } 1112 1113 /* 1114 * Process a complete RX transaction. 1115 */ 1116 1117 STATIC void 1118 rx_trans_process(int cpu, 1119 int client_id, 1120 _atp_client_t *client, 1121 _atp_rx_trans_t *rx_trans) 1122 { 1123 bcm_rx_t rv = BCM_RX_HANDLED; 1124 bcm_pkt_t *pkt = NULL; 1125 uint8 *payload = NULL; 1126 int do_callback = TRUE; 1127 1128 if (rx_trans->flags & _ATP_RX_F_LOOPBACK) { 1129 LOG_DEBUG(BSL_LS_TKS_ATP, 1130 (BSL_META("ATP RX: Loopback packet cli %d\n"), 1131 client->client_id)); 1132 payload = rx_trans->lb_data + CPUTRANS_HEADER_BYTES; 1133 } else { /* Non-loopback; check for reassembly and multiple segments */ 1134 pkt = rx_trans->pkt; 1135 if (client->flags & ATP_F_REASSEM_BUF) { 1136 pkt->alloc_ptr = _pkt_reassem(rx_trans); 1137 if (pkt->alloc_ptr == NULL) { 1138 LOG_WARN(BSL_LS_TKS_ATP, 1139 (BSL_META("ATP RX: Failed to alloc for reassem\n"))); 1140 do_callback = FALSE; 1141 } else { /* alloc pointer is set */ 1142 payload = (uint8 *)pkt->alloc_ptr + CPUTRANS_HEADER_BYTES; 1143 } 1144 } else { 1145 if (rx_trans->_atp_hdr.tot_segs == 1) { 1146 payload = pkt->pkt_data[1].data; 1147 } else { /* Multiple segments, indicate w/ NULL payload */ 1148 payload = NULL; 1149 } 1150 } 1151 } 1152 1153 if (do_callback) { 1154 if (client->callback != NULL) { 1155 #if defined(BROADCOM_DEBUG) && defined(ATP_LONG_CALLBACK_TRACKER) 1156 atp_rx_in = sal_time_usecs(); 1157 atp_rx_out = 0; 1158 atp_rx_ptr = client->callback; 1159 #endif /* BROADCOM_DEBUG */ 1160 rv = client->callback(CPU_KEY(cpu), 1161 client_id, 1162 pkt, 1163 payload, 1164 rx_trans->payload_len, 1165 client->cookie); 1166 #if defined(BROADCOM_DEBUG) && defined(ATP_LONG_CALLBACK_TRACKER) 1167 atp_rx_out = sal_time_usecs(); 1168 if (atp_rx_out - atp_rx_in > 1000000) { 1169 ++atp_rx_long_callbacks; 1170 if (atp_rx_long_cb_ptr == NULL) { 1171 atp_rx_long_cb_ptr = atp_rx_ptr; 1172 } 1173 } 1174 #endif /* BROADCOM_DEBUG */ 1175 } 1176 } 1177 1178 if (rv == BCM_RX_HANDLED_OWNED) { 1179 rx_trans->lb_data = NULL; 1180 rx_trans->pkt = NULL; 1181 } else { 1182 /* The packet is not stolen, free it now. */ 1183 if (rx_trans->flags & _ATP_RX_F_LOOPBACK) { 1184 _atp_trans_ptr->tp_data_free(_atp_trans_ptr->tp_unit, 1185 rx_trans->lb_data); 1186 rx_trans->lb_data = NULL; 1187 } else { 1188 _atp_rx_pkt_free(rx_trans->pkt); 1189 } 1190 } 1191 1192 rx_trans->pkt = NULL; 1193 rx_trans->flags |= _ATP_RX_F_HANDLED; 1194 } 1195 1196 STATIC int 1197 rx_stale(_atp_rx_trans_t *_trans, sal_usecs_t _cur_time) 1198 { 1199 int rv = FALSE; 1200 1201 if (!(_trans->flags & _ATP_RX_F_DATA_READY)) { 1202 int dt = SAL_USECS_SUB(_cur_time, _trans->rcvd_time); 1203 rv = ((dt < 0) || (dt > (4 * ATP_TIMEOUT))); 1204 } 1205 1206 return rv; 1207 1208 } 1209 1210 1211 STATIC void 1212 _atp_rcv_data_queue_process(_atp_client_t *client, int cpu) 1213 { 1214 _atp_rx_trans_t *rx_trans; 1215 1216 rx_trans = client->cpu[cpu].rx_trans; 1217 1218 if (rx_trans != NULL) { 1219 sal_usecs_t cur_time; 1220 _atp_rx_trans_t *rx_trans_next; 1221 1222 cur_time = sal_time_usecs(); 1223 1224 while (rx_trans != NULL) { 1225 1226 rx_trans_next = rx_trans->next; 1227 1228 /* Mark stale RX transactions */ 1229 if (rx_stale(rx_trans, cur_time)) { 1230 INCR_COUNTER(stale_rx_trans); 1231 atp_rx_trans_delete(rx_trans); 1232 } else { 1233 if (rx_trans->flags & _ATP_RX_F_DATA_READY && 1234 !(rx_trans->flags & _ATP_RX_F_HANDLED)) { 1235 rx_trans_process(cpu, client->client_id, client, rx_trans); 1236 } 1237 1238 if (rx_trans->flags & _ATP_RX_F_HANDLED) { 1239 atp_rx_trans_delete(rx_trans); 1240 } 1241 } 1242 rx_trans = rx_trans_next; 1243 } 1244 } 1245 } 1246 1247 /* 1248 * Go through all RX transactions; send up the stack if ready to go. 1249 */ 1250 1251 STATIC void 1252 rx_callbacks_check(void) 1253 { 1254 int idx; 1255 _atp_client_t *cli; 1256 int cpu; 1257 1258 ATP_RX_LOCK; 1259 FOREACH_CLIENT(cli, idx) { 1260 for (cpu = 0; cpu < atp_cpu_max; cpu++) { 1261 _atp_rcv_data_queue_process(cli, cpu); 1262 } 1263 } 1264 ATP_RX_UNLOCK; 1265 } 1266 1267 /* pkt_buf points to start of L2 header */ 1268 1269 STATIC bcm_rx_t 1270 _atp_rx_pkt_process(int src_cpu, uint8 *pkt_buf, int pkt_len) 1271 { 1272 _atp_hdr_t _atp_hdr; 1273 bcm_rx_t rv = BCM_RX_NOT_HANDLED; 1274 1275 _atp_hdr_unpack(pkt_buf, &_atp_hdr); 1276 1277 if (!(_atp_hdr.hdr_flags & _ATP_HDR_NO_ACK)) { 1278 rv = atp_data_handle(src_cpu, pkt_buf, pkt_len, &_atp_hdr); 1279 } else { 1280 rv = bet_data_handle(src_cpu, pkt_buf, pkt_len, &_atp_hdr); 1281 } 1282 1283 #if defined(BCM_RXP_DEBUG) 1284 if (rv == BCM_RX_HANDLED_OWNED) { 1285 bcm_rx_pool_own(pkt_buf, "atp_pkt_proc"); 1286 } 1287 #endif 1288 return rv; 1289 } 1290 1291 STATIC int 1292 _handle_rx_data(_atp_pkt_data_t *pkt_p) 1293 { 1294 cpudb_key_t src_key; 1295 int src_cpu; 1296 uint8 *pkt_buf; 1297 int len; 1298 1299 pkt_buf = pkt_p->pkt_buf; 1300 1301 /* Look for the source CPU */ 1302 CPUDB_KEY_UNPACK(&pkt_buf[CPUTRANS_SRC_KEY_OFS], src_key); 1303 src_cpu = _atp_key_lookup(src_key); 1304 if (src_cpu < 0) { 1305 if (_atp_flags & ATP_F_LEARN_SLF) { 1306 src_cpu = _atp_key_add(src_key); 1307 if (src_cpu < 0) { 1308 LOG_VERBOSE(BSL_LS_TKS_ATP, 1309 (BSL_META("ATP pkt: could not add key\n"))); 1310 ++mem_rx_drop; 1311 return BCM_RX_HANDLED; 1312 } 1313 } else { 1314 LOG_VERBOSE(BSL_LS_TKS_ATP, 1315 (BSL_META("ATP pkt: SLF drop\n"))); 1316 ++slf_rx_drop; 1317 return BCM_RX_HANDLED; 1318 } 1319 } 1320 1321 /* Process the packet */ 1322 len = pkt_p->len; 1323 1324 return _atp_rx_pkt_process(src_cpu, pkt_buf, len); 1325 } 1326 1327 STATIC void 1328 rx_process_pkt_data(void) 1329 { 1330 _atp_pkt_data_t *cur_p; 1331 _atp_pkt_data_t *next_p; 1332 int rv; 1333 1334 ATP_RX_LOCK; 1335 ATP_PKT_DATA_LOCK; /* Steal the current queue of pkts */ 1336 cur_p = _atp_rcv_data_queue; 1337 _atp_rcv_data_queue_tail = NULL; 1338 _atp_rcv_data_queue = NULL; 1339 ATP_PKT_DATA_UNLOCK; 1340 1341 while (cur_p != NULL) { 1342 next_p = cur_p->next; 1343 1344 rv = _handle_rx_data(cur_p); 1345 if (rv != BCM_RX_HANDLED_OWNED) { /* Free packet data */ 1346 _atp_trans_ptr->tp_data_free(_atp_trans_ptr->tp_unit, 1347 cur_p->pkt_buf); 1348 } 1349 ATP_PKT_DATA_LOCK; /* Free the pkt struct */ 1350 cur_p->next = _atp_pkt_data_freelist; 1351 _atp_pkt_data_freelist = cur_p; 1352 ATP_PKT_DATA_UNLOCK; 1353 1354 cur_p = next_p; 1355 } 1356 ATP_RX_UNLOCK; 1357 } 1358 1359 /**************************************************************** 1360 * 1361 * ATP RX Thread 1362 */ 1363 1364 STATIC void 1365 atp_rx_thread(void *cookie) 1366 { 1367 COMPILER_REFERENCE(cookie); 1368 1369 LOG_VERBOSE(BSL_LS_TKS_ATP, 1370 (BSL_META("ATP: RX Thread starting\n"))); 1371 atp_rx_thread_exit = FALSE; 1372 while (1) { 1373 int sleep_time = sal_sem_FOREVER; 1374 1375 sal_sem_take(atp_rx_sem, sleep_time); 1376 if (atp_rx_thread_exit) { /* Exit forced */ 1377 break; 1378 } 1379 1380 rx_process_pkt_data(); 1381 rx_callbacks_check(); 1382 1383 } 1384 1385 atp_rx_thread_id = SAL_THREAD_ERROR; 1386 LOG_VERBOSE(BSL_LS_TKS_ATP, 1387 (BSL_META("ATP: RX Thread exiting\n"))); 1388 sal_thread_exit(0); 1389 } 1390 1391 /**************************************************************** 1392 * 1393 * Packet handling routines 1394 */ 1395 1396 /* RX trans create assumes ATP_RX_LOCK is held and if loopback is false, 1397 _atp_hdr is valid 1398 */ 1399 1400 STATIC _atp_rx_trans_t * 1401 atp_rx_trans_create(_atp_client_t *client, int src_cpu, uint8 *pkt_buf, 1402 int pkt_len, _atp_hdr_t *_atp_hdr, int loopback) 1403 { 1404 _atp_rx_trans_t *rx_trans; 1405 bcm_pkt_t *pkt; 1406 int seg; 1407 1408 LOG_DEBUG(BSL_LS_TKS_ATP, 1409 (BSL_META("ATP new RX, cpu %d, new seq %d, cli %d, old seq %d," 1410 "flags: %sseen\n"), 1411 src_cpu, (_atp_hdr != NULL) ? _atp_hdr->seq_num : -1, 1412 client->client_id, 1413 client->cpu[src_cpu].rx_seq_num, 1414 client->cpu[src_cpu].cpu_flags & ATP_CPU_RX_TRANS_SEEN ? 1415 "" : "not ")); 1416 if (!loopback) { 1417 if (_atp_hdr->segment >= _atp_hdr->tot_segs) { 1418 LOG_WARN(BSL_LS_TKS_ATP, 1419 (BSL_META("ATP RX: Bad seg num %d >= tot %d.\n"), 1420 _atp_hdr->segment, _atp_hdr->tot_segs)); 1421 return NULL; 1422 } 1423 } 1424 1425 /* Grab a transaction from the free list */ 1426 rx_trans = NULL; 1427 ATP_RXQ_LOCK; 1428 if (rx_trans_freelist != NULL) { 1429 rx_trans = (_atp_rx_trans_t *)rx_trans_freelist; 1430 rx_trans_freelist = rx_trans->next; 1431 } 1432 ATP_RXQ_UNLOCK; 1433 1434 if (rx_trans == NULL) { 1435 LOG_VERBOSE(BSL_LS_TKS_ATP, 1436 (BSL_META("ATP RX freelist empty\n"))); 1437 INCR_COUNTER(rx_trans_fail); 1438 return NULL; 1439 } 1440 INCR_COUNTER(rxt_create); 1441 1442 rx_trans->next = NULL; 1443 rx_trans->flags = 0; 1444 rx_trans->client = client; 1445 rx_trans->src_cpu = src_cpu; 1446 rx_trans->rcvd_time = sal_time_usecs(); 1447 rx_trans->lb_data = NULL; 1448 1449 if (loopback) { 1450 rx_trans->flags = _ATP_RX_F_DATA_READY | _ATP_RX_F_LOOPBACK; 1451 rx_trans->lb_data = pkt_buf; 1452 rx_trans->pkt = NULL; 1453 rx_trans->payload_len = pkt_len - CPUTRANS_HEADER_BYTES; 1454 LOG_DEBUG(BSL_LS_TKS_ATP, 1455 (BSL_META("RT create LB %p\n"), 1456 rx_trans)); 1457 return rx_trans; 1458 } 1459 1460 /* Payload length doesn't include CPUTRANS header */ 1461 rx_trans->payload_len = pkt_len - CPUTRANS_HEADER_BYTES; 1462 1463 /* Not a loopback packet; Copy the header, and init members */ 1464 sal_memcpy((void *)&rx_trans->_atp_hdr, _atp_hdr, sizeof(_atp_hdr_t)); 1465 rx_trans->ack_count = 0; 1466 rx_trans->rcv_segs = 1; 1467 1468 /* Allocate a receive pkt w/ enough pointers: 1469 * Use 2 pointers per segment and link 1470 * headers in pointers 0, 2, 4... and data in ptrs 1, 3, 5... 1471 */ 1472 pkt = rx_trans->pkt = cputrans_rx_pkt_alloc(2 * _atp_hdr->tot_segs); 1473 if (pkt == NULL) { 1474 LOG_ERROR(BSL_LS_TKS_ATP, 1475 (BSL_META("ATP RX Could not allocate packet\n"))); 1476 1477 /* Return to freelist; transaction not yet queued */ 1478 atp_rx_trans_delete(rx_trans); 1479 INCR_COUNTER(rxt_pkt_alloc_fail); 1480 return NULL; 1481 } 1482 pkt->alloc_ptr = NULL; 1483 1484 /* Put pkt header in first block and payload in second block */ 1485 seg = _atp_hdr->segment; 1486 SET_PKT_BLK_DATA(seg, pkt, pkt_buf, pkt_len); 1487 1488 if (_atp_hdr->tot_segs == 1) { 1489 rx_trans->flags |= _ATP_RX_F_DATA_READY; 1490 } 1491 1492 LOG_DEBUG(BSL_LS_TKS_ATP, 1493 (BSL_META("RT create %p\n"), 1494 rx_trans)); 1495 return rx_trans; 1496 } 1497 1498 1499 /* 1500 * Free the BET RX packet controlled by client, for given source CPU. 1501 * Allocation pointers are stored in the even numbered blocks. 1502 * err indicates if dropping counts as an error. 1503 */ 1504 1505 STATIC void 1506 bet_rx_pkt_drop(_atp_client_t *client, int src_cpu, int err) 1507 { 1508 bcm_pkt_t *pkt; 1509 int count; 1510 bcm_pkt_blk_t *pkt_blk; 1511 int i; 1512 1513 pkt = client->cpu[src_cpu].bet_rx_pkts; 1514 client->cpu[src_cpu].bet_rx_pkts = NULL; 1515 1516 if (pkt != NULL) { 1517 count = pkt->blk_count; 1518 pkt_blk = pkt->pkt_data; 1519 for (i = 0; i < count; i += 2) { 1520 if (pkt_blk[i].data != NULL) { 1521 _atp_trans_ptr->tp_data_free(_atp_trans_ptr->tp_unit, 1522 pkt_blk[i].data); 1523 pkt_blk[i].data = NULL; 1524 } 1525 } 1526 cputrans_rx_pkt_free(pkt); 1527 if (err) { 1528 ++bet_rx_drop; 1529 } 1530 } 1531 } 1532 1533 /* Allows ACKs to be sent asynchronously */ 1534 1535 STATIC void 1536 atp_rx_ack_cb(uint8 *pkt_buf, void *cookie) 1537 { 1538 _atp_client_cpu_t *cp; 1539 1540 COMPILER_REFERENCE(pkt_buf); 1541 1542 cp = (_atp_client_cpu_t *)cookie; 1543 cp->cpu_flags &= ~ATP_CPU_ACK_PENDING; 1544 } 1545 1546 STATIC void 1547 atp_rx_ack_nh_cb(int rv, uint8 *pkt_buf, void *cookie) 1548 { 1549 _atp_client_cpu_t *cp; 1550 1551 COMPILER_REFERENCE(pkt_buf); 1552 COMPILER_REFERENCE(rv); 1553 1554 cp = (_atp_client_cpu_t *)cookie; 1555 cp->cpu_flags &= ~ATP_CPU_ACK_PENDING; 1556 } 1557 1558 /* 1559 * Send an ACK for the client/cpu; rx_trans MAY BE NULL 1560 */ 1561 1562 STATIC void 1563 atp_rx_ack(_atp_client_t *cli, int cpu, _atp_rx_trans_t *rx_trans, 1564 uint16 seq_num, 1565 uint8 *pkt_buf, int pkt_len) 1566 { 1567 int ack_bytes = 0; 1568 bcm_pkt_t *pkt; 1569 int i; 1570 int rv; 1571 int cos; 1572 int next_hop; 1573 uint32 ct_flags; 1574 _atp_client_cpu_t *cp; 1575 uint8 *ack_data; 1576 next_hop_tx_callback_f ack_nh_cb; 1577 c2c_cb_f ack_c2c_cb; 1578 1579 if (rx_trans == NULL) { /* ACK entire frame */ 1580 ack_bytes = 0; /* Signals all done */ 1581 ack_data = cli->cpu[cpu].rx_ack_data; 1582 } else { /* RX trans there */ 1583 if (RX_TRANS_DATA_DONE(rx_trans) || rx_trans->pkt == NULL) { 1584 ack_bytes = rx_trans->payload_len; 1585 } else { 1586 pkt = rx_trans->pkt; 1587 for (i = 1; i < pkt->blk_count; i += 2) { 1588 if (pkt->pkt_data[i].data != NULL) { 1589 ack_bytes += pkt->pkt_data[i].len; 1590 } else { /* Break on first unknown data */ 1591 break; 1592 } 1593 } 1594 if (ack_bytes == 0) { 1595 /* Don't ack 0-bytes here because that would imply all done */ 1596 return; 1597 } 1598 } 1599 ack_data = rx_trans->ack_data; 1600 } 1601 1602 cp = &cli->cpu[cpu]; 1603 cp->atp_hdr.seq_num = seq_num; 1604 cp->atp_hdr.opcode = _ATP_OPC_ACK; 1605 cp->atp_hdr.start_byte = ack_bytes; 1606 cos = cli->cos; 1607 next_hop = cp->atp_hdr.hdr_flags & _ATP_HDR_NEXT_HOP; 1608 _atp_hdr_pack(ack_data, (_atp_hdr_t *)&cp->atp_hdr); 1609 1610 /* 1611 * If ACK is to be sent immediately, do not set callbacks. 1612 * Otherwise, set corresponding rx ack callback routines. 1613 */ 1614 if (cp->atp_hdr.hdr_flags & _ATP_HDR_IMMEDIATE_ACK) { 1615 ack_nh_cb = NULL; 1616 ack_c2c_cb = NULL; 1617 } else { 1618 ack_nh_cb = atp_rx_ack_nh_cb; 1619 ack_c2c_cb = atp_rx_ack_cb; 1620 cp->cpu_flags |= ATP_CPU_ACK_PENDING; 1621 } 1622 1623 1624 LOG_DEBUG(BSL_LS_TKS_ATP, 1625 (BSL_META("Sending ACK to %d cli %d seq %d bytes %d\n"), 1626 cpu, 1627 cli->client_id, cp->atp_hdr.seq_num, cp->atp_hdr.start_byte)); 1628 1629 /* To ensure proper TX pkt allocation */ 1630 ct_flags = CPUTRANS_NO_HEADER_ALLOC; 1631 CPUTRANS_COS_SET(ct_flags, cos); /* Only cos is needed */ 1632 1633 if (next_hop) { 1634 rv = next_hop_tx(ack_data, 1635 CPUTRANS_HEADER_BYTES, 1636 cos, 1637 cli->vlan, 1638 _atp_seg_len, 1639 ct_flags, 1640 ATP_PKT_TYPE, 1641 CPU_KEY(cpu), 1642 ack_nh_cb, 1643 (void *)cp); 1644 } else { 1645 rv = c2c_tx(CPU_KEY(cpu), 1646 ack_data, 1647 CPUTRANS_HEADER_BYTES, 1648 cos, 1649 cli->vlan, 1650 CPUTRANS_HEADER_BYTES, /* Force one segment */ 1651 ATP_PKT_TYPE, 1652 ct_flags, 1653 ack_c2c_cb, 1654 (void *)cp); 1655 } 1656 1657 if (rv != BCM_E_NONE) { 1658 #ifdef BROADCOM_DEBUG 1659 { 1660 int proto = ((pkt_buf[52]<<8) | pkt_buf[53]); /* Always Big Endian */ 1661 LOG_VERBOSE(BSL_LS_TKS_ATP, 1662 (BSL_META("ATP: Failed sending ACK to %x:%x for protocol %d (%d): %s\n"), 1663 pkt_buf[10], pkt_buf[11], proto, rv, bcm_errmsg(rv))); 1664 } 1665 #endif /* BROADCOM_DEBUG */ 1666 cp->cpu_flags &= ~ATP_CPU_ACK_PENDING; 1667 } else if (rx_trans != NULL) { 1668 rx_trans->ack_count++; 1669 } 1670 } 1671 1672 /* 1673 * new_trans is a new RX transaction; verify tail_trans is done and update 1674 * client pointers. Assumes ATP_RX_LOCK is held. 1675 */ 1676 STATIC bcm_rx_t 1677 new_rx_trans_add(_atp_client_t *client, _atp_hdr_t *_atp_hdr, 1678 _atp_rx_trans_t *new_trans, int src_cpu, 1679 uint8 *pkt_buf, int pkt_len) 1680 { 1681 _atp_rx_trans_t *tail_trans; 1682 1683 tail_trans = client->cpu[src_cpu].rx_tail; 1684 if (tail_trans != NULL) { 1685 /* New transaction; previous pkt should be done */ 1686 if (!(tail_trans->flags & _ATP_RX_F_DATA_READY)) { 1687 /* Drop tail packet by marking all done */ 1688 LOG_WARN(BSL_LS_TKS_ATP, 1689 (BSL_META("ATP RX Dropping non-ready rx trans, cpu %d, " 1690 "client %d, seq %d\n"), src_cpu, client->client_id, 1691 _atp_hdr->seq_num)); 1692 tail_trans->flags |= _ATP_RX_F_DATA_READY | _ATP_RX_F_HANDLED; 1693 atp_rx_drop++; 1694 } 1695 } 1696 1697 /* Indicate transaction seen */ 1698 client->cpu[src_cpu].cpu_flags |= ATP_CPU_RX_TRANS_SEEN; 1699 client->cpu[src_cpu].rx_seq_num = _atp_hdr->seq_num; 1700 1701 /* 1702 * If sequence number is > 0, indicate a conversation has 1703 * started with this CPU 1704 */ 1705 if (_atp_hdr->seq_num > 0) { 1706 _atp_cpu_info[src_cpu].flags |= _ATP_RX_CXN_INIT; 1707 } 1708 1709 sal_memcpy((_atp_hdr_t *)&client->cpu[src_cpu].atp_hdr, _atp_hdr, 1710 sizeof(_atp_hdr_t)); 1711 1712 atp_rx_ack(client, src_cpu, new_trans, _atp_hdr->seq_num, 1713 pkt_buf, pkt_len); 1714 1715 atp_rx_trans_enqueue(new_trans, client, src_cpu); 1716 1717 return BCM_RX_HANDLED_OWNED; 1718 } 1719 1720 /* 1721 * Update the current transaction (tail of queue) in cur_trans with 1722 * info from _atp_hdr. TAIL must not be NULL on entry. 1723 */ 1724 1725 STATIC bcm_rx_t 1726 current_rx_trans_update(_atp_client_t *client, 1727 int src_cpu, 1728 _atp_hdr_t *_atp_hdr, 1729 uint8 *pkt_buf, 1730 int pkt_len) 1731 1732 { 1733 int seg_idx; 1734 bcm_rx_t rv = BCM_RX_HANDLED; 1735 bcm_pkt_blk_t *pkt_blks; 1736 _atp_rx_trans_t *cur_trans = NULL; 1737 1738 /* On going transaction */ 1739 LOG_DEBUG(BSL_LS_TKS_ATP, 1740 (BSL_META("ATP Ongoing RX: client %d. cpu %d. seq %d\n"), 1741 client->client_id, src_cpu, client->cpu[src_cpu].rx_seq_num)); 1742 if (_atp_hdr->segment >= _atp_hdr->tot_segs) { 1743 LOG_WARN(BSL_LS_TKS_ATP, 1744 (BSL_META("ATP RX: Bad seg num %d >= tot %d.\n"), 1745 _atp_hdr->segment, _atp_hdr->tot_segs)); 1746 return BCM_RX_HANDLED; 1747 } 1748 1749 cur_trans = client->cpu[src_cpu].rx_tail; /* May be NULL */ 1750 if ((cur_trans != NULL) && 1751 (!(cur_trans->flags & _ATP_RX_F_DATA_READY))) { 1752 /* Add data to this transaction */ 1753 seg_idx = _atp_hdr->segment; 1754 pkt_blks = cur_trans->pkt->pkt_data; 1755 1756 if (pkt_blks[2 * seg_idx].data == NULL) { /* New data */ 1757 LOG_DEBUG(BSL_LS_TKS_ATP, 1758 (BSL_META("ATP new data\n"))); 1759 SET_PKT_BLK_DATA(seg_idx, cur_trans->pkt, pkt_buf, pkt_len); 1760 cur_trans->payload_len += 1761 pkt_len - CPUTRANS_HEADER_BYTES; 1762 if (++cur_trans->rcv_segs == _atp_hdr->tot_segs) { 1763 /* Packet is complete */ 1764 cur_trans->flags |= _ATP_RX_F_DATA_READY; 1765 } 1766 rv = BCM_RX_HANDLED_OWNED; 1767 } else { 1768 LOG_DEBUG(BSL_LS_TKS_ATP, 1769 (BSL_META("ATP old data\n"))); 1770 rv = BCM_RX_HANDLED; 1771 } 1772 } 1773 1774 /* Always ACK the data */ 1775 atp_rx_ack(client, src_cpu, cur_trans, _atp_hdr->seq_num, 1776 pkt_buf, pkt_len); 1777 1778 return rv; 1779 } 1780 1781 1782 /**************************************************************** 1783 * 1784 * ATP handle data ACK 1785 */ 1786 1787 /* Handle an ATP ACK */ 1788 STATIC bcm_rx_t 1789 atp_ack_handle(int src_cpu, _atp_hdr_t *_atp_hdr) 1790 { 1791 _atp_tx_trans_t *trans; 1792 _atp_client_t *client; 1793 uint16 new_seq_num; 1794 1795 new_seq_num = _atp_hdr->seq_num; 1796 LOG_DEBUG(BSL_LS_TKS_ATP, 1797 (BSL_META("ATP ACK from %d cli %d. seq %d. bytes %d. tot %d\n"), 1798 src_cpu, 1799 _atp_hdr->client_id, new_seq_num, _atp_hdr->start_byte, 1800 _atp_hdr->tot_bytes)); 1801 1802 /* Look for the TX operation this is ACK-ing */ 1803 client = client_find(_atp_hdr->client_id); 1804 if (client == NULL) { 1805 INCR_COUNTER(invalid_client_cnt); 1806 LOG_WARN(BSL_LS_TKS_ATP, 1807 (BSL_META("ATP: ACK on NULL client %d\n"), 1808 _atp_hdr->client_id)); 1809 return BCM_RX_HANDLED; 1810 } 1811 1812 trans = client->cpu[src_cpu].tx_trans; 1813 while (trans != NULL) { 1814 if (new_seq_num == trans->_atp_hdr.seq_num) { 1815 break; /* Found ack'd transaction. */ 1816 } 1817 trans = trans->next; 1818 } 1819 1820 /* 1821 * At this point, only care if the ACK number == transaction number 1822 * and this is the first time the ACK has been seen. 1823 */ 1824 if (trans != NULL && 1825 (new_seq_num == trans->_atp_hdr.seq_num) && 1826 !(trans->flags & _ATP_TX_F_DONE)) { 1827 if (_atp_hdr->start_byte == 0 || /* 0 means all ACK'd */ 1828 _atp_hdr->start_byte >= _atp_hdr->tot_bytes) { 1829 LOG_DEBUG(BSL_LS_TKS_ATP, 1830 (BSL_META("ATP ACK SN %d marking done.\n"), 1831 new_seq_num)); 1832 trans->flags |= _ATP_TX_F_DONE; 1833 } else if (trans->bytes_acked < _atp_hdr->start_byte) { 1834 trans->bytes_acked = _atp_hdr->start_byte; 1835 } 1836 } else { /* Redundant ACK */ 1837 LOG_DEBUG(BSL_LS_TKS_ATP, 1838 (BSL_META("ATP extra ACK SN %d (trans SN %d) from %d\n"), 1839 new_seq_num, 1840 trans != NULL ? trans->_atp_hdr.seq_num : -1, src_cpu)); 1841 } 1842 1843 return BCM_RX_HANDLED; 1844 } 1845 1846 /**************************************************************** 1847 * ATP handle packet data. Create a new transaction if 1848 * necessary. If transaction exists, check if this data 1849 * segment is new. Indicate if ACK needed and wake thread. 1850 */ 1851 1852 STATIC bcm_rx_t 1853 atp_data_handle(int src_cpu, uint8 *pkt_buf, int pkt_len, 1854 _atp_hdr_t *_atp_hdr) 1855 { 1856 _atp_rx_trans_t *new_trans = NULL; 1857 _atp_client_t *client; 1858 bcm_rx_t rv = BCM_RX_HANDLED; 1859 uint16 new_seq_num; 1860 int16 seq_num_diff; /* Used signed value of same size for difference */ 1861 1862 new_seq_num = _atp_hdr->seq_num; 1863 1864 client = client_find(_atp_hdr->client_id); 1865 if (client == NULL) { 1866 INCR_COUNTER(invalid_client_cnt); 1867 LOG_VERBOSE(BSL_LS_TKS_ATP, 1868 (BSL_META("ATP rx: Unknown client id %d\n"), 1869 _atp_hdr->client_id)); 1870 return BCM_RX_NOT_HANDLED; 1871 } 1872 1873 if (!CPU_VALID_IDX(src_cpu)) { 1874 LOG_VERBOSE(BSL_LS_TKS_ATP, 1875 (BSL_META("ATP rx: invalid cpu index %d\n"), 1876 src_cpu)); 1877 return BCM_RX_NOT_HANDLED; 1878 } 1879 1880 /* Protect this queue while updating */ 1881 1882 seq_num_diff = new_seq_num - client->cpu[src_cpu].rx_seq_num; 1883 1884 /* First check for reset condition: Seq num goes back to 0 after 1885 * RX_CXN_INIT is set (that is, after other transactions seen). 1886 * This is only allowed on the first packet to avoid the race 1887 * condition of a retransmit on the first packet. 1888 */ 1889 if ((_atp_cpu_info[src_cpu].flags & _ATP_RX_CXN_INIT) && 1890 (new_seq_num == 0) && 1891 (!(_atp_hdr->hdr_flags & _ATP_HDR_RETRANSMIT))) { 1892 /* Special case indicating remote reset; clear CPU state */ 1893 LOG_VERBOSE(BSL_LS_TKS_ATP, 1894 (BSL_META("ATP: New CPU data for cpu %d, cli %d\n"), 1895 src_cpu, 1896 _atp_hdr->client_id)); 1897 _atp_cpu_purge(src_cpu); 1898 } else { 1899 if ((seq_num_diff > 1) && (new_seq_num > 1)) { 1900 LOG_VERBOSE(BSL_LS_TKS_ATP, 1901 (BSL_META("ATP Warning: seq num jump cpu %d, cli %d: new %d. " 1902 "diff %d.\n"), src_cpu, client->client_id, new_seq_num, 1903 seq_num_diff)); 1904 } 1905 if ((!(client->cpu[src_cpu].cpu_flags & ATP_CPU_RX_TRANS_SEEN)) || 1906 (seq_num_diff != 0) || 1907 (new_seq_num == 0) || 1908 (new_seq_num == 1)) { 1909 /* New transaction */ 1910 new_trans = atp_rx_trans_create(client, src_cpu, pkt_buf, 1911 pkt_len, _atp_hdr, FALSE); 1912 if (new_trans == NULL) { 1913 LOG_VERBOSE(BSL_LS_TKS_ATP, 1914 (BSL_META("ATP Could not allocate new RX transaction\n"))); 1915 } else { 1916 rv = new_rx_trans_add(client, _atp_hdr, new_trans, src_cpu, 1917 pkt_buf, pkt_len); 1918 } 1919 } else if (seq_num_diff == 0) { 1920 rv = current_rx_trans_update(client, src_cpu, _atp_hdr, 1921 pkt_buf, pkt_len); 1922 } else { /* seq_num_diff < 0; old transaction, bad data */ 1923 LOG_VERBOSE(BSL_LS_TKS_ATP, 1924 (BSL_META("ATP: old RX data SN %d. Diff %d. cpu %d\n"), 1925 new_seq_num, 1926 seq_num_diff, 1927 /* coverity[dead_error_begin] */ 1928 src_cpu)); 1929 ++old_rx_trans_drop; 1930 } 1931 } 1932 return rv; 1933 } 1934 1935 STATIC uint8 * 1936 _bet_pkt_reassem(_atp_client_t *client, int src_cpu, int *tot_len) 1937 { 1938 bcm_pkt_t *pkt; 1939 int i; 1940 uint8 *pkt_buf; 1941 int pi = 0; /* packet buffer offset */ 1942 1943 /* Find the total size needed and allocate the buffer */ 1944 pkt = client->cpu[src_cpu].bet_rx_pkts; 1945 *tot_len = 0; 1946 for (i = 1; i < pkt->blk_count; i += 2) { 1947 *tot_len += pkt->pkt_data[i].len; 1948 } 1949 pkt_buf = NULL; 1950 _atp_trans_ptr->tp_data_alloc(_atp_trans_ptr->tp_unit, 1951 *tot_len, 0, (void*)&pkt_buf); 1952 if (pkt_buf == NULL) { 1953 INCR_COUNTER(reassem_alloc_fail); 1954 return NULL; 1955 } 1956 1957 /* Gather up the segments into the new buffer */ 1958 for (i = 1; i < pkt->blk_count; i += 2) { 1959 sal_memcpy(&pkt_buf[pi], pkt->pkt_data[i].data, pkt->pkt_data[i].len); 1960 pi += pkt->pkt_data[i].len; 1961 } 1962 1963 return pkt_buf; 1964 } 1965 1966 /* 1967 * Check a multisegment BET packet for consistency with what's being 1968 * received; create new accumulation packet if necessary. 1969 */ 1970 1971 STATIC bcm_rx_t 1972 bet_rx_multi_seg_check(_atp_client_t *client, int src_cpu, _atp_hdr_t *_atp_hdr, 1973 uint8 *pkt_buf, int len, int *make_callback) 1974 { 1975 int seg_idx; 1976 bcm_pkt_t *pkt; 1977 int make_new_pkt = FALSE; 1978 bcm_rx_t rv = BCM_RX_HANDLED; 1979 1980 *make_callback = FALSE; 1981 1982 seg_idx = _atp_hdr->segment; /* This pkt's segment ID */ 1983 pkt = client->cpu[src_cpu].bet_rx_pkts; /* Current pkt accumulating */ 1984 if (pkt != NULL) { 1985 if (_atp_hdr->seq_num != BET_RX_SEQ_NUM(pkt)) { 1986 /* New packet doesn't match current sequence number; drop 1987 accumulated packets. 1988 */ 1989 bet_rx_pkt_drop(client, src_cpu, TRUE); 1990 make_new_pkt = TRUE; 1991 } else if ((2 * seg_idx) >= pkt->blk_count) { 1992 LOG_WARN(BSL_LS_TKS_ATP, 1993 (BSL_META("ATP BET bad segment index\n"))); 1994 bet_rx_pkt_drop(client, src_cpu, TRUE); 1995 } else { 1996 if (pkt->pkt_data[2 * seg_idx].data == NULL) { /* New segment */ 1997 SET_PKT_BLK_DATA(seg_idx, pkt, pkt_buf, len); 1998 } 1999 2000 if (packet_data_done(pkt)) { 2001 *make_callback = TRUE; 2002 } 2003 rv = BCM_RX_HANDLED_OWNED; 2004 } 2005 } else { /* No current packet */ 2006 make_new_pkt = TRUE; 2007 } 2008 2009 if (make_new_pkt) { 2010 if (seg_idx > 0) { /* Dump the packet; didn't get seg 0 */ 2011 rv = BCM_RX_HANDLED; 2012 } else { 2013 /* Alloc new pkt and copy data pointers to this pkt */ 2014 pkt = client->cpu[src_cpu].bet_rx_pkts = 2015 cputrans_rx_pkt_alloc(2 * _atp_hdr->tot_segs); 2016 if (pkt == NULL) { 2017 INCR_COUNTER(rx_mseg_alloc_fail); 2018 rv = BCM_RX_HANDLED; 2019 } else { 2020 SET_PKT_BLK_DATA(seg_idx, pkt, pkt_buf, len); 2021 BET_RX_SEQ_NUM_SET(pkt, _atp_hdr->seq_num); 2022 rv = BCM_RX_HANDLED_OWNED; 2023 } 2024 } 2025 } 2026 2027 return rv; 2028 } 2029 2030 /* 2031 * Handle Best Effort data; assumes client_index set in _atp_hdr 2032 * This must support src_cpu < 0 for single segment packets. 2033 */ 2034 2035 STATIC bcm_rx_t 2036 bet_data_handle(int src_cpu, uint8 *pkt_buf, int len, _atp_hdr_t *_atp_hdr) 2037 { 2038 bcm_rx_t rv = BCM_RX_HANDLED; 2039 _atp_client_t *client; 2040 int make_callback = FALSE; 2041 int cb_len = 0; 2042 int i; 2043 2044 /* The callback information */ 2045 uint8 *cb_pkt_payload = NULL; 2046 bcm_pkt_t *cb_pkt_ptr = NULL; 2047 2048 /* Find the client pointer for this pkt */ 2049 client = client_find(_atp_hdr->client_id); 2050 if (client == NULL) { 2051 INCR_COUNTER(invalid_client_cnt); 2052 LOG_VERBOSE(BSL_LS_TKS_ATP, 2053 (BSL_META("BET rx: Unknown client id %d\n"), 2054 _atp_hdr->client_id)); 2055 return BCM_RX_NOT_HANDLED; 2056 } 2057 2058 if (!CPU_VALID_IDX(src_cpu)) { 2059 LOG_VERBOSE(BSL_LS_TKS_ATP, 2060 (BSL_META("BET rx: invalid cpu index %d\n"), 2061 src_cpu)); 2062 return BCM_RX_NOT_HANDLED; 2063 } 2064 2065 /* Is there more than one segment for this packet? */ 2066 if (_atp_hdr->tot_segs > 1) { /* Yes, multiple segments */ 2067 rv = bet_rx_multi_seg_check(client, src_cpu, _atp_hdr, pkt_buf, len, 2068 &make_callback); 2069 if (make_callback) { 2070 /* Check if accumulation necessary */ 2071 if (client->flags & ATP_F_REASSEM_BUF) { 2072 cb_pkt_payload = _bet_pkt_reassem(client, src_cpu, &cb_len); 2073 if (cb_pkt_payload == NULL) { 2074 make_callback = FALSE; 2075 } /* else, make_callback is already true */ 2076 } else { 2077 cb_pkt_ptr = client->cpu[src_cpu].bet_rx_pkts; 2078 /* Calculate payload length of packet */ 2079 for (i = 1; i < cb_pkt_ptr->blk_count; i += 2) { 2080 cb_len += cb_pkt_ptr->pkt_data[i].len; 2081 } 2082 } 2083 } 2084 } else { /* Single segment for the packet; make callback w/ data */ 2085 make_callback = TRUE; 2086 cb_pkt_payload = pkt_buf + CPUTRANS_HEADER_BYTES; 2087 cb_len = len - CPUTRANS_HEADER_BYTES; 2088 } 2089 2090 if (make_callback) { 2091 if (client->callback != NULL) { 2092 rv = client->callback(CPU_KEY(src_cpu), 2093 _atp_hdr->client_id, 2094 cb_pkt_ptr, 2095 cb_pkt_payload, 2096 cb_len, 2097 client->cookie); 2098 } 2099 2100 if (_atp_hdr->tot_segs > 1) { 2101 /* Multi segment BET */ 2102 2103 if (rv == BCM_RX_HANDLED_OWNED) { 2104 /* 2105 Callback owns the buffers. If reassembly was 2106 requested, the callback owns cb_pkt_payload, and 2107 cb_pkt_ptr is NULL. If reassembly was *not* 2108 requested, cb_pkt_payload is NULL, and the callback 2109 owns cb_pkt_ptr. 2110 */ 2111 2112 if (cb_pkt_ptr) { 2113 /* Callback owns packet - remove reference */ 2114 client->cpu[src_cpu].bet_rx_pkts = NULL; 2115 } else { 2116 /* Reassembly - release accumulator */ 2117 bet_rx_pkt_drop(client, src_cpu, FALSE); 2118 } 2119 } else { /* Callback does not own the buffer(s) */ 2120 2121 bet_rx_pkt_drop(client, src_cpu, FALSE); 2122 2123 if (cb_pkt_payload) { 2124 /* Free reassembly buffer */ 2125 _atp_trans_ptr->tp_data_free(_atp_trans_ptr->tp_unit, 2126 cb_pkt_payload); 2127 } 2128 } 2129 2130 /* 2131 In all final states of a multi-segment BET packet, the 2132 buffer passed to this function is either owned by the 2133 callback, or has been dropped by bet_rx_pkt_drop(), so 2134 claim ownership of the buffer. 2135 */ 2136 rv = BCM_RX_HANDLED_OWNED; 2137 } 2138 2139 /* For single segment BET, the ownership of the buffer is 2140 completely determined by the return value of the 2141 callback. */ 2142 } 2143 2144 return rv; 2145 } 2146 2147 2148 /* 2149 * Enqueue the packet data for later handling. Currently, we 2150 * assume a single packet buffer and that the first 4 bytes 2151 * can be used as a "next" pointer for the linked list. 2152 */ 2153 2154 STATIC int 2155 enqueue_atp_data(uint8 *pkt_buf, int len, uint32 flags) 2156 { 2157 _atp_pkt_data_t *atp_pkt; 2158 uint8 opcode; 2159 2160 /* Check opcode; and split into TX related data and RX related data */ 2161 opcode = _atp_opcode_get(pkt_buf); 2162 switch (opcode) { 2163 case _ATP_OPC_DATA: /* ATP data */ 2164 ATP_PKT_DATA_LOCK; 2165 if (_atp_pkt_data_freelist == NULL) { /* No free space in rx queue */ 2166 ATP_PKT_DATA_UNLOCK; 2167 INCR_COUNTER(rx_pkt_drops); 2168 return -1; 2169 } 2170 atp_pkt = _atp_pkt_data_freelist; 2171 _atp_pkt_data_freelist = _atp_pkt_data_freelist->next; 2172 atp_pkt->len = len; 2173 atp_pkt->pkt_buf = pkt_buf; 2174 atp_pkt->flags = flags; 2175 if (_atp_rcv_data_queue_tail == NULL) { /* Queue is empty */ 2176 _atp_rcv_data_queue = atp_pkt; 2177 } else { 2178 _atp_rcv_data_queue_tail->next = atp_pkt; 2179 } 2180 _atp_rcv_data_queue_tail = atp_pkt; 2181 atp_pkt->next = NULL; 2182 ATP_PKT_DATA_UNLOCK; 2183 ATP_RX_THREAD_WAKE; 2184 break; 2185 2186 case _ATP_OPC_ACK: /* ACK for packet we sent */ 2187 ATP_PKT_DATA_LOCK; 2188 if (_atp_pkt_data_freelist == NULL) { /* No free space in rx queue */ 2189 ATP_PKT_DATA_UNLOCK; 2190 INCR_COUNTER(ack_pkt_drops); 2191 return -1; 2192 } 2193 atp_pkt = _atp_pkt_data_freelist; 2194 _atp_pkt_data_freelist = _atp_pkt_data_freelist->next; 2195 atp_pkt->len = len; 2196 atp_pkt->pkt_buf = pkt_buf; 2197 atp_pkt->flags = flags; 2198 if (_atp_trx_data_queue_tail == NULL) { /* Queue is empty */ 2199 _atp_trx_data_queue = atp_pkt; 2200 } else { 2201 _atp_trx_data_queue_tail->next = atp_pkt; 2202 } 2203 _atp_trx_data_queue_tail = atp_pkt; 2204 atp_pkt->next = NULL; 2205 ATP_PKT_DATA_UNLOCK; 2206 ATP_TX_THREAD_WAKE; 2207 break; 2208 2209 default: /* Uh oh. */ 2210 LOG_ERROR(BSL_LS_TKS_ATP, 2211 (BSL_META("ATP: Bad packet opcode: %d\n"), 2212 opcode)); 2213 return -1; 2214 } 2215 2216 2217 return 0; 2218 } 2219 2220 /* This routine is registered with BCM RX to receive C2C packets */ 2221 2222 STATIC bcm_rx_t 2223 _atp_rx_callback(int unit, bcm_pkt_t *pkt, void *cookie) 2224 { 2225 uint8 *pkt_buf; 2226 uint16 mplx_num; 2227 cpudb_key_t src_key; 2228 int len; 2229 2230 COMPILER_REFERENCE(cookie); 2231 2232 if (!_atp_running) { 2233 INCR_COUNTER(atp_not_running); 2234 LOG_DEBUG(BSL_LS_TKS_ATP, 2235 (BSL_META_U(unit, 2236 "ATP pkt in, not running\n"))); 2237 return BCM_RX_NOT_HANDLED; 2238 } 2239 2240 pkt_buf = pkt->pkt_data[0].data; 2241 if (c2c_pkt_recognize(pkt_buf, &src_key, &mplx_num) != BCM_E_NONE) { 2242 return BCM_RX_NOT_HANDLED; 2243 } 2244 2245 /* Local ID field matches ATP? */ 2246 if (mplx_num != ATP_PKT_TYPE) { 2247 LOG_DEBUG(BSL_LS_TKS_ATP, 2248 (BSL_META_U(unit, 2249 "ATP pkt in, bad pkt type %d\n"), 2250 mplx_num)); 2251 return BCM_RX_NOT_HANDLED; 2252 } 2253 2254 len = pkt->pkt_len; 2255 2256 /* Strip CRC if not already done by receiver */ 2257 if (!(pkt->flags & BCM_RX_CRC_STRIP)) { 2258 len -= 4; 2259 } 2260 if (enqueue_atp_data(pkt_buf, len, pkt->flags) < 0) { 2261 return BCM_RX_HANDLED; 2262 } 2263 2264 2265 return BCM_RX_HANDLED_OWNED; 2266 } 2267 2268 /* This routine is registered with Next Hop to receive (next hop) 2269 packets. Next Hop packets should not have a CRC at the end, so no 2270 need to handle here. 2271 */ 2272 2273 STATIC bcm_rx_t 2274 _atp_next_hop_callback(cpudb_key_t src_key, int mplx_num, int unit, int port, 2275 uint8* pkt_buf, int len, void *cookie) 2276 { 2277 COMPILER_REFERENCE(cookie); 2278 COMPILER_REFERENCE(mplx_num); 2279 2280 if (!_atp_running) { 2281 INCR_COUNTER(atp_not_running); 2282 LOG_DEBUG(BSL_LS_TKS_ATP, 2283 (BSL_META_U(unit, 2284 "BETNH pkt in, not running\n"))); 2285 return BCM_RX_NOT_HANDLED; 2286 } 2287 2288 if (enqueue_atp_data(pkt_buf, len, 0) < 0) { 2289 return BCM_RX_HANDLED; 2290 } 2291 2292 return BCM_RX_HANDLED_OWNED; 2293 } 2294 2295 /**************************************************************** 2296 * 2297 * TX handling 2298 */ 2299 2300 /* 2301 * Create a TX transaction. Sets up all the bcm_pkt_t structs necessary 2302 * as a linked list. Assumes TX lock held. 2303 * 2304 * CPUTRANS_COS_OVERRIDE and CPUTRANS_INT_PRIO_OVERRIDE in ct_flags 2305 * override default cos and internal priority values. 2306 */ 2307 2308 STATIC _atp_tx_trans_t * 2309 _atp_tx_trans_create(int dest_cpu, 2310 _atp_client_t *cli, 2311 int no_ack, 2312 uint8 *pkt_buf, 2313 int len, 2314 uint32 ct_flags, 2315 atp_tx_cb_f cb, 2316 void *cookie) 2317 { 2318 _atp_tx_trans_t *trans; 2319 bcm_pkt_t *pkt, *cur_pkt; 2320 int tot_segs; 2321 int tot_bytes; 2322 _atp_hdr_t *_atp_hdr; 2323 int i; 2324 int cos; 2325 int next_hop; 2326 int immediate_ack; 2327 int rv; 2328 2329 tot_segs = 0; 2330 tot_bytes = len; 2331 if (ct_flags & CPUTRANS_NO_HEADER_ALLOC) { 2332 tot_bytes -= CPUTRANS_HEADER_BYTES; 2333 } 2334 2335 if (tot_bytes > ATP_MTU) { 2336 LOG_ERROR(BSL_LS_TKS_ATP, 2337 (BSL_META("ATP TX: Packet too big (%d bytes)\n"), 2338 tot_bytes)); 2339 return NULL; 2340 } 2341 2342 trans = tx_trans_freelist; 2343 if (trans == NULL) { /* Allocation failed */ 2344 INCR_COUNTER(tx_trans_fail); 2345 return NULL; 2346 } 2347 tx_trans_freelist = trans->next; 2348 INCR_COUNTER(txt_create); 2349 2350 /* Is client or TX operation next hop? (or broadcast?) */ 2351 next_hop = CLI_IS_NEXT_HOP(cli) || (ct_flags & CPUTRANS_NEXT_HOP) || 2352 (ct_flags & CPUTRANS_BCAST); 2353 2354 /* Is ack to be sent immediately? */ 2355 immediate_ack = ct_flags & CPUTRANS_IMMEDIATE_ACK; 2356 2357 /* Default COS comes from client; flags may override */ 2358 cos = cli->cos; 2359 if (ct_flags & CPUTRANS_COS_OVERRIDE) { 2360 CPUTRANS_COS_SET(cos, CPUTRANS_COS_GET(ct_flags)); 2361 } else { 2362 /* To ensure proper TX pkt allocation */ 2363 CPUTRANS_COS_SET(ct_flags, cos); /* Only cos is needed */ 2364 } 2365 2366 /* If internal priority is provided in flag, override default value */ 2367 if (ct_flags & CPUTRANS_INT_PRIO_OVERRIDE) { 2368 CPUTRANS_INT_PRIO_SET(cos, CPUTRANS_INT_PRIO_GET(ct_flags)); 2369 } 2370 2371 sal_memset((void *)trans, 0, sizeof(_atp_tx_trans_t)); 2372 trans->seg_len = _atp_seg_len; 2373 trans->next = NULL; 2374 2375 trans->db_update = atp_db_update_count; 2376 /* Set up the transaction; note CPU_KEY is bcast if dest invalid */ 2377 if (next_hop) { 2378 pkt = trans->pkt_list = 2379 next_hop_pkt_create(pkt_buf, 2380 len, 2381 cos, 2382 cli->vlan, 2383 trans->seg_len, 2384 ct_flags, 2385 ATP_PKT_TYPE, 2386 CPU_KEY(dest_cpu), 2387 &tot_segs, 2388 &rv); 2389 } else { /* c2c packet */ 2390 pkt = trans->pkt_list = 2391 c2c_pkt_create(CPU_KEY(dest_cpu), 2392 pkt_buf, 2393 len, 2394 cos, 2395 cli->vlan, 2396 trans->seg_len, 2397 ATP_PKT_TYPE, 2398 ct_flags, 2399 &tot_segs, 2400 &rv); 2401 } 2402 2403 if (rv != BCM_E_NONE) { /* Failed to allocate */ 2404 atp_tx_trans_delete(trans); 2405 INCR_COUNTER(txt_pkt_alloc_fail); 2406 return NULL; 2407 } 2408 2409 /* Setup transaction and header */ 2410 trans->client = cli; 2411 trans->callback = cb; 2412 trans->cookie = cookie; 2413 trans->pkt_buf = pkt_buf; 2414 trans->len = len; 2415 trans->dest_cpu = dest_cpu; 2416 trans->ct_flags = ct_flags; 2417 2418 _atp_hdr = (_atp_hdr_t *)&(trans->_atp_hdr); 2419 _atp_hdr->client_id = cli->client_id; 2420 2421 _atp_hdr->tot_bytes = tot_bytes; 2422 _atp_hdr->tot_segs = tot_segs; 2423 _atp_hdr->opcode = _ATP_OPC_DATA; 2424 _atp_hdr->cos = CPUTRANS_COS_GET(cos); /* Only cos */ 2425 if (next_hop) { 2426 _atp_hdr->hdr_flags |= _ATP_HDR_NEXT_HOP; 2427 } 2428 if (no_ack) { 2429 _atp_hdr->hdr_flags |= _ATP_HDR_NO_ACK; 2430 } 2431 if (immediate_ack) { 2432 _atp_hdr->hdr_flags |= _ATP_HDR_IMMEDIATE_ACK; 2433 } 2434 2435 /* Pack the headers for each segment */ 2436 for (i = 0, cur_pkt = pkt; cur_pkt != NULL; cur_pkt = cur_pkt->next) { 2437 _atp_hdr->segment = i++; 2438 _atp_hdr_pack(cur_pkt->pkt_data[0].data, (_atp_hdr_t *)&trans->_atp_hdr); 2439 } 2440 _atp_hdr->segment = 0; 2441 2442 LOG_DEBUG(BSL_LS_TKS_ATP, 2443 (BSL_META("TT create %p\n"), 2444 trans)); 2445 return trans; 2446 } 2447 2448 2449 /* Loopback an ATP packet */ 2450 STATIC int 2451 _atp_loopback(_atp_client_t *client, int local_cpu, 2452 uint8 *pkt_buf, int len, uint32 ct_flags) 2453 { 2454 _atp_rx_trans_t *new_trans = NULL; 2455 uint8 *rx_buf; 2456 uint8 *payload = pkt_buf; 2457 int alloc_len = len; 2458 2459 LOG_DEBUG(BSL_LS_TKS_ATP, 2460 (BSL_META("ATP TX: Loopback packet len %d, " 2461 "flgs 0x%x\n"), 2462 len, ct_flags)); 2463 2464 if (ct_flags & CPUTRANS_NO_HEADER_ALLOC) { 2465 payload += CPUTRANS_HEADER_BYTES; 2466 } else { /* Need header space for consistency on free */ 2467 alloc_len += CPUTRANS_HEADER_BYTES; 2468 } 2469 2470 /* 2471 * Create an RX transaction and enqueue, sync'd w/ RX thread. 2472 * The buffer is copied so that the TX can return before the RX 2473 * handles the buffer. To be consistent with buffer reassembly, 2474 * a NULL pointer is added to the start of the data. 2475 * 2476 * ATP header bytes are present in RX buffer for consistency 2477 */ 2478 rx_buf = NULL; 2479 _atp_trans_ptr->tp_data_alloc(_atp_trans_ptr->tp_unit, 2480 alloc_len, 0, (void*)&rx_buf); 2481 if (rx_buf == NULL) { 2482 INCR_COUNTER(lb_buf_alloc_fail); 2483 LOG_WARN(BSL_LS_TKS_ATP, 2484 (BSL_META("ATP LB Could not alloc RX pkt\n"))); 2485 return BCM_E_RESOURCE; 2486 } 2487 sal_memcpy(rx_buf + CPUTRANS_HEADER_BYTES, payload, len); 2488 2489 new_trans = atp_rx_trans_create(client, local_cpu, rx_buf, 2490 alloc_len, NULL, TRUE); 2491 if (new_trans == NULL) { 2492 _atp_trans_ptr->tp_data_free(_atp_trans_ptr->tp_unit, rx_buf); 2493 LOG_WARN(BSL_LS_TKS_ATP, 2494 (BSL_META("ATP LB Could not alloc RX trans %d\n"), 2495 alloc_len)); 2496 return BCM_E_RESOURCE; 2497 } 2498 2499 /* Enqueue the transaction and wake the RX thread */ 2500 atp_rx_trans_enqueue(new_trans, client, local_cpu); 2501 sal_sem_give(atp_rx_sem); 2502 2503 return BCM_E_NONE; 2504 } 2505 2506 /**************************************************************** 2507 * 2508 * ATP API 2509 */ 2510 2511 2512 /* 2513 * Function: 2514 * atp_config_done 2515 * Purpose: 2516 * Indicate if ATP is set up 2517 * Returns: 2518 * TRUE if configured and running. 2519 */ 2520 2521 int 2522 atp_running(void) 2523 { 2524 return _atp_running; 2525 } 2526 2527 2528 /* 2529 * Function: 2530 * atp_config_get 2531 * Purpose: 2532 * Get the configurable parameters for ATP 2533 * Parameters: 2534 * tx_thrd_pri - (OUT) The priority to use when starting ATP thread 2535 * rx_thrd_pri - (OUT) The priority to use when starting ATP thread 2536 * trans_ptr - (OUT) Pointer to transport struct 2537 * Returns: 2538 * BCM_E_XXX 2539 */ 2540 2541 int 2542 atp_config_get(int *tx_thrd_pri, int *rx_thrd_pri, 2543 bcm_trans_ptr_t **trans_ptr) 2544 { 2545 if (tx_thrd_pri != NULL) { 2546 *tx_thrd_pri = atp_tx_thread_priority; 2547 } 2548 if (rx_thrd_pri != NULL) { 2549 *rx_thrd_pri = atp_rx_thread_priority; 2550 } 2551 if (trans_ptr != NULL) { 2552 *trans_ptr = _atp_trans_ptr; 2553 } 2554 2555 return BCM_E_NONE; 2556 } 2557 2558 2559 /* 2560 * Function: 2561 * atp_config_set 2562 * Purpose: 2563 * Set the configurable parameters for ATP 2564 * Parameters: 2565 * tx_thrd_pri - The priority to use for ATP TX thread 2566 * rx_thrd_pri - The priority to use for ATP RX thread 2567 * trans_ptr - Pointer to transport struct 2568 * Returns: 2569 * BCM_E_XXX 2570 * Notes: 2571 * thread priorities are ignored if < 0; trans_ptr is ignored if NULL. 2572 */ 2573 2574 int 2575 atp_config_set(int tx_thrd_pri, int rx_thrd_pri, 2576 bcm_trans_ptr_t *trans_ptr) 2577 { 2578 if (tx_thrd_pri >= 0) { 2579 atp_tx_thread_priority = tx_thrd_pri; 2580 } 2581 if (rx_thrd_pri >= 0) { 2582 atp_rx_thread_priority = rx_thrd_pri; 2583 } 2584 if (trans_ptr != NULL) { 2585 if (trans_ptr->tp_data_alloc == NULL || 2586 trans_ptr->tp_data_free == NULL) { 2587 return BCM_E_PARAM; 2588 } 2589 _atp_trans_ptr = trans_ptr; 2590 } 2591 2592 return BCM_E_NONE; 2593 } 2594 2595 2596 /* 2597 * Function: 2598 * atp_start 2599 * Purpose: 2600 * Start the ATP thread and register with RX 2601 * Parameters: 2602 * flags - See atp.h; mainly, learn on src miss 2603 * unit_bmp - bitmap of units on which to register with RX 2604 * rco_flags - BCM RX callout flags. See include/bcm/rx.h. 2605 * Returns: 2606 * BCM_E_XXX 2607 * Notes: 2608 * If desired, atp_config_set should be called before this to 2609 * set transport pointer, etc. 2610 */ 2611 2612 int 2613 atp_start(uint32 flags, 2614 uint32 unit_bmp, 2615 uint32 rco_flags) 2616 { 2617 int rv; 2618 2619 if (_atp_running) { 2620 INCR_COUNTER(atp_not_running); 2621 return BCM_E_BUSY; 2622 } 2623 2624 ATP_INIT_CHECK; /* Call init if that's not done yet */ 2625 2626 /* Register to receive next hop packets for ATP */ 2627 if (!next_hop_running()) { 2628 LOG_WARN(BSL_LS_TKS_ATP, 2629 (BSL_META("ATP Warning: next hop is not running\n"))); 2630 } 2631 rv = next_hop_register(_atp_next_hop_callback, NULL, ATP_PKT_TYPE); 2632 if (rv < 0) { 2633 LOG_WARN(BSL_LS_TKS_ATP, 2634 (BSL_META("ATP Warning: cannot register with next hop\n"))); 2635 } 2636 2637 /* Start the threads */ 2638 atp_tx_thread_id = sal_thread_create("bcmATP-TX", 2639 SAL_THREAD_STKSZ, 2640 atp_tx_thread_priority, 2641 atp_tx_thread, NULL); 2642 if (atp_tx_thread_id == SAL_THREAD_ERROR) { 2643 return BCM_E_MEMORY; 2644 } 2645 atp_rx_thread_id = sal_thread_create("bcmATP-RX", 2646 SAL_THREAD_STKSZ, 2647 atp_rx_thread_priority, 2648 atp_rx_thread, NULL); 2649 if (atp_rx_thread_id == SAL_THREAD_ERROR) { 2650 sal_thread_destroy(atp_tx_thread_id); 2651 atp_tx_thread_id = SAL_THREAD_ERROR; 2652 return BCM_E_MEMORY; 2653 } 2654 2655 /* 2656 * Register to receive packets from active units 2657 */ 2658 rv = cputrans_rx_bmp_register(unit_bmp, "atp", _atp_rx_callback, 2659 ATP_RX_PRIORITY, NULL, 2660 rco_flags); 2661 if (rv != BCM_E_NONE) { 2662 LOG_WARN(BSL_LS_TKS_ATP, 2663 (BSL_META("ATP Warning: Could not register RX %d: %s\n"), 2664 rv, bcm_errmsg(rv))); 2665 } 2666 2667 _atp_flags = flags; 2668 _atp_units = unit_bmp; 2669 _atp_running = TRUE; 2670 2671 LOG_VERBOSE(BSL_LS_TKS_ATP, 2672 (BSL_META("ATP: Started\n"))); 2673 2674 return BCM_E_NONE; 2675 } 2676 2677 2678 /* 2679 * Function: 2680 * atp_stop 2681 * Purpose: 2682 * Stop the ATP thread, unregister and clean up. 2683 * Parameters: 2684 * 2685 * Returns: 2686 * BCM_E_XXX 2687 * Notes: 2688 */ 2689 2690 int 2691 atp_stop(void) 2692 { 2693 cputrans_rx_bmp_unregister(_atp_units, _atp_rx_callback, ATP_RX_PRIORITY); 2694 2695 ATP_LOCK; 2696 _atp_running = FALSE; 2697 atp_cleanup(); 2698 ATP_UNLOCK; 2699 2700 LOG_VERBOSE(BSL_LS_TKS_ATP, 2701 (BSL_META("ATP: Stopped\n"))); 2702 2703 return BCM_E_NONE; 2704 } 2705 2706 2707 /* 2708 * Function: 2709 * atp_cos_vlan_set 2710 * Purpose: 2711 * Set the default cos, internal priority, and vlan for clients 2712 * Parameters: 2713 * cos - COS and internal priority value to use as default; 2714 * if < 0, do not change 2715 * vlan - VLAN value to use as default if valid 2716 * Returns: 2717 * BCM_E_NONE 2718 * 2719 * The 'cos' parameter contains the cos and internal priority 2720 * values encoded. Use CPUTRANS_COS_SET() CPUTRANS_INT_PRIO_SET() 2721 * to set values accordingly. The internal priority is optional; 2722 * if this is not provided, the cos value is used for internal priority. 2723 */ 2724 2725 int 2726 atp_cos_vlan_set(int cos, int vlan) 2727 { 2728 if (cos >= 0) { 2729 _atp_cos = CPUTRANS_COS_GET(cos); 2730 if (cos & CPUTRANS_INT_PRIO_VALID) { 2731 CPUTRANS_INT_PRIO_SET(_atp_cos, CPUTRANS_INT_PRIO_GET(cos)); 2732 } 2733 } 2734 2735 if (vlan >= 0 && vlan < 4096) { 2736 _atp_vlan = vlan; 2737 } 2738 2739 return BCM_E_NONE; 2740 } 2741 2742 2743 /* 2744 * Function: 2745 * atp_cos_vlan_get 2746 * Purpose: 2747 * Get the default cos, internal priority, and vlan for clients 2748 * Parameters: 2749 * cos - (OUT) Pointer where to store cos/internal priority value 2750 * vlan - (OUT) Pointer where to store vlan value 2751 * Returns: 2752 * BCM_E_NONE 2753 * Notes: 2754 * Pointers may be NULL; then ignored. 2755 * 2756 * The 'cos' parameter contains the cos and internal priority 2757 * values encoded. Use CPUTRANS_COS_SET() CPUTRANS_INT_PRIO_SET() 2758 * to set values accordingly. The internal priority is optional; 2759 * if this is not provided, the cos value is used for internal priority. 2760 */ 2761 2762 int 2763 atp_cos_vlan_get(int *cos, int *vlan) 2764 { 2765 if (cos != NULL) { 2766 *cos = _atp_cos; 2767 } 2768 if (vlan != NULL) { 2769 *vlan = _atp_vlan; 2770 } 2771 2772 return BCM_E_NONE; 2773 } 2774 2775 2776 /* 2777 * Function: 2778 * atp_db_update 2779 * Purpose: 2780 * Update ATP with the indicated CPU database 2781 * Parameters: 2782 * db_ref - The new database 2783 * Returns: 2784 * BCM_E_XXX 2785 * Notes: 2786 */ 2787 2788 2789 STATIC int 2790 atp_db_update_locked(cpudb_ref_t db_ref) 2791 { 2792 cpudb_entry_t *entry; 2793 int idx; 2794 2795 for (idx = 0; idx < CPUDB_CPU_MAX; idx++) { 2796 if (CPU_VALID(idx)) { 2797 CPUDB_KEY_SEARCH(db_ref, _atp_cpu_info[idx].key, entry); 2798 if (entry == NULL) { 2799 next_hop_key_invalidate(_atp_cpu_info[idx].key); 2800 _atp_cpu_remove(idx); 2801 } 2802 } 2803 } 2804 if (db_ref->num_cpus > CPUDB_CPU_MAX) { 2805 return BCM_E_MEMORY; 2806 } 2807 CPUDB_FOREACH_ENTRY(db_ref, entry) { 2808 if (_atp_key_lookup(entry->base.key) == -1) { 2809 if (_atp_key_add(entry->base.key) < 0) { 2810 return BCM_E_RESOURCE; 2811 } 2812 } 2813 } 2814 atp_db_update_count++; 2815 return BCM_E_NONE; 2816 } 2817 2818 int 2819 atp_db_update(cpudb_ref_t db_ref) 2820 { 2821 int rv; 2822 2823 /* Remove CPUs not present in DB */ 2824 ATP_INIT_CHECK; /* Call init if that's not done yet */ 2825 ATP_LOCK; 2826 rv = atp_db_update_locked(db_ref); 2827 ATP_UNLOCK; 2828 2829 if (rv == BCM_E_NONE) { 2830 LOG_VERBOSE(BSL_LS_TKS_ATP, 2831 (BSL_META("ATP: Updating c2c db\n"))); 2832 rv = c2c_cpu_update(db_ref); 2833 } else if (rv == BCM_E_MEMORY) { 2834 LOG_ERROR(BSL_LS_TKS_ATP, 2835 (BSL_META("ATP ERROR: Too many CPUs in DB\n"))); 2836 } else if (rv == BCM_E_RESOURCE) { 2837 LOG_WARN(BSL_LS_TKS_ATP, 2838 (BSL_META("ATP WARN: Failed to add CPU key\n"))); 2839 } 2840 return rv; 2841 } 2842 2843 #define FREE_CHECK(id) \ 2844 do { \ 2845 if ((id) != NULL) { \ 2846 sal_free((void *)id); \ 2847 (id) = NULL; \ 2848 } \ 2849 } while (0) 2850 2851 /* End threads if possible */ 2852 2853 STATIC int 2854 _atp_end_threads(int retries) 2855 { 2856 int rv = BCM_E_NONE; 2857 int i; 2858 2859 if (atp_tx_thread_id != SAL_THREAD_ERROR || 2860 atp_rx_thread_id != SAL_THREAD_ERROR) { 2861 atp_tx_thread_exit = TRUE; 2862 atp_rx_thread_exit = TRUE; 2863 ATP_TX_THREAD_WAKE; 2864 ATP_RX_THREAD_WAKE; 2865 /* Allow thread to exit */ 2866 for (i = 0; i < retries; i++) { 2867 if (atp_tx_thread_id == SAL_THREAD_ERROR && 2868 atp_rx_thread_id == SAL_THREAD_ERROR) { 2869 break; 2870 } 2871 sal_usleep(10000); 2872 } 2873 if (atp_rx_thread_id != SAL_THREAD_ERROR || 2874 atp_tx_thread_id != SAL_THREAD_ERROR) { 2875 if (atp_tx_thread_id != SAL_THREAD_ERROR) { 2876 LOG_WARN(BSL_LS_TKS_ATP, 2877 (BSL_META("Warning: ATP TX thread did not exit\n"))); 2878 } 2879 if (atp_rx_thread_id != SAL_THREAD_ERROR) { 2880 LOG_WARN(BSL_LS_TKS_ATP, 2881 (BSL_META("Warning: ATP RX thread did not exit\n"))); 2882 } 2883 return BCM_E_FAIL; 2884 } 2885 } 2886 2887 return rv; 2888 } 2889 2890 /* 2891 * Clean up the ATP subsystem. 2892 * Assumes lock is held. Does not remove existing clients, but clears 2893 * out their pending transactions. 2894 */ 2895 2896 STATIC void 2897 atp_cleanup(void) 2898 { 2899 int i; 2900 _atp_client_t *client, *next_cli; 2901 int cpu; 2902 2903 if (!init_done) { 2904 return; 2905 } 2906 2907 (void)_atp_end_threads(50); 2908 sal_thread_yield(); 2909 2910 /* Release any pending TX sync ops */ 2911 FOREACH_CLIENT(client, i) { 2912 for (cpu = 0; cpu < CPUDB_CPU_MAX; cpu++) { 2913 atp_tx_trans_delete_all(client, cpu); 2914 atp_rx_trans_delete_all(client, cpu); 2915 } 2916 } 2917 2918 /* Clear all clients */ 2919 for (i = 0; i < _ATP_CLIENT_HASH_MAX; i++) { 2920 client = _atp_client_buckets[i]; 2921 while (client != NULL) { 2922 next_cli = client->next; 2923 (void)client_delete(client, FALSE); 2924 client = next_cli; 2925 } 2926 } 2927 2928 for (i = 0; i < CPUDB_CPU_MAX; i++) { 2929 _atp_cpu_remove(i); 2930 } 2931 2932 if (ack_pkt_data != NULL) { 2933 atp_ack_pkt_data_from_heap ? 2934 sal_free(ack_pkt_data) : sal_dma_free(ack_pkt_data); 2935 ack_pkt_data = NULL; 2936 } 2937 2938 init_done = FALSE; 2939 } 2940 2941 #undef FREE_CHECK 2942 2943 /**************************************************************** 2944 * 2945 * ATP configuration functions 2946 */ 2947 2948 2949 /* 2950 * Function: 2951 * atp_timeout_set 2952 * Purpose: 2953 * Set the timeout parameters for an ATP transmit 2954 * Parameters: 2955 * retry_usecs - The timeout value in microseconds 2956 * num_retries - The number of retry attempts 2957 * Returns: 2958 * BCM_E_XXX 2959 * Notes: 2960 * The system clock usually has a minimal tick size of about 10000 2961 * usecs. 2962 * 2963 * Checks the parameters against minimum allowable 2964 */ 2965 2966 int 2967 atp_timeout_set(int retry_usecs, int num_retries) 2968 { 2969 if (retry_usecs < ATP_RETRY_TIMEOUT_MIN) { 2970 LOG_WARN(BSL_LS_TKS_ATP, 2971 (BSL_META("ATP Warning: changing retry timeout " 2972 "from %d to %d\n"), retry_usecs, ATP_RETRY_TIMEOUT_MIN)); 2973 atp_retry_timeout = ATP_RETRY_TIMEOUT_MIN; 2974 } else { 2975 atp_retry_timeout = retry_usecs; 2976 } 2977 2978 if (num_retries < 1) { 2979 LOG_WARN(BSL_LS_TKS_ATP, 2980 (BSL_META("ATP Warning: changing retry count from " 2981 "%d to %d\n"), num_retries, 1)); 2982 atp_retry_count = 1; 2983 } else { 2984 atp_retry_count = num_retries; 2985 } 2986 2987 return BCM_E_NONE; 2988 } 2989 2990 2991 /* 2992 * Function: 2993 * atp_timeout_get 2994 * Purpose: 2995 * Get the timeout parameters for an ATP transmit 2996 * Parameters: 2997 * retry_usecs - (OUT) The timeout value in microseconds 2998 * num_retries - (OUT) The number of retry attempts 2999 * Returns: 3000 * BCM_E_XXX 3001 * Notes: 3002 */ 3003 3004 int 3005 atp_timeout_get(int *retry_usecs, int *num_retries) 3006 { 3007 *retry_usecs = atp_retry_timeout; 3008 *num_retries = atp_retry_count; 3009 3010 return BCM_E_NONE; 3011 } 3012 3013 3014 /* 3015 * Function: 3016 * atp_timeout_register 3017 * Purpose: 3018 * Set the timeout callback function that will be called 3019 * when an ATP transmission fails due to timeout. 3020 * Parameters: 3021 * callback - The callback function 3022 * (or NULL for no callback) 3023 * Returns: 3024 * BCM_E_XXX 3025 */ 3026 int 3027 atp_timeout_register(atp_timeout_cb_f callback) 3028 { 3029 BASE_INIT_CHECK; 3030 3031 ATP_LOCK; 3032 atp_timeout_cb = callback; 3033 ATP_UNLOCK; 3034 3035 return BCM_E_NONE; 3036 } 3037 3038 3039 /* 3040 * Function: 3041 * atp_segment_len_set 3042 * Purpose: 3043 * Set the segmentation length 3044 * Parameters: 3045 * seg_len - Max length of a segment in bytes 3046 * Returns: 3047 * BCM_E_XXX 3048 * Notes: 3049 * Checks the parameter against minimum allowable 3050 */ 3051 3052 #define SEG_LEN_MIN 16 /* Let's be reasonable here..... */ 3053 3054 int 3055 atp_segment_len_set(int seg_len) 3056 { 3057 int rv = BCM_E_NONE; 3058 3059 BASE_INIT_CHECK; 3060 ATP_LOCK; 3061 3062 if (seg_len < SEG_LEN_MIN) { 3063 rv = BCM_E_PARAM; 3064 } else { 3065 _atp_seg_len = seg_len; 3066 } 3067 3068 ATP_UNLOCK; 3069 return rv; 3070 } 3071 3072 3073 /* 3074 * Function: 3075 * atp_segment_len_get 3076 * Purpose: 3077 * Get the segmentation length 3078 * Returns: 3079 * Current segment length 3080 */ 3081 3082 int 3083 atp_segment_len_get(void) 3084 { 3085 return _atp_seg_len; 3086 } 3087 3088 3089 /* 3090 * Function: 3091 * atp_pool_size_set 3092 * Purpose: 3093 * Set the maximum allowed number of TX/RX transaction buffers 3094 * Parameters: 3095 * tx_max - Max number of tx transaction buffers 3096 * rx_max - Max number of rx transaction buffers 3097 * Returns: 3098 * BCM_E_XXX 3099 * Notes: 3100 * Applies to TX and RX independently. 3101 * Effects will not take affect until restart. 3102 * 3103 * If parameter is < 0, use the default value. 3104 */ 3105 3106 int 3107 atp_pool_size_set(int tx_max, int rx_max) 3108 { 3109 if (tx_max < 0) { 3110 atp_tx_pool_size = ATP_TX_TRANSACT_DEFAULT; 3111 } else { 3112 atp_tx_pool_size = tx_max; 3113 } 3114 3115 if (rx_max < 0) { 3116 atp_rx_pool_size = ATP_RX_TRANSACT_DEFAULT; 3117 } else { 3118 atp_rx_pool_size = rx_max; 3119 } 3120 3121 return BCM_E_NONE; 3122 } 3123 3124 3125 /* 3126 * Function: 3127 * atp_pool_size_get 3128 * Purpose: 3129 * Get the current max pending operations setting 3130 * Parameters: 3131 * tx_max - (OUT) Max number of tx transaction buffers 3132 * rx_max - (OUT) Max number of rx transaction buffers 3133 * Returns: 3134 * BCM_E_XXX 3135 * Notes: 3136 */ 3137 3138 int 3139 atp_pool_size_get(int *tx_max, int *rx_max) 3140 { 3141 *tx_max = atp_tx_pool_size; 3142 *rx_max = atp_rx_pool_size; 3143 3144 return BCM_E_NONE; 3145 } 3146 3147 3148 /**************************************************************** 3149 * 3150 * Init and clean up functions 3151 */ 3152 3153 #define ALLOC_CHECK(id, bytes, rv) \ 3154 do { \ 3155 (id) = sal_alloc((bytes), "ATP"); \ 3156 if ((id) == NULL) { \ 3157 atp_cleanup(); \ 3158 ATP_UNLOCK; \ 3159 return rv; \ 3160 } \ 3161 sal_memset((void *)(id), 0, bytes); \ 3162 } while (0) 3163 3164 #define RX_ALLOC_CHECK(id, bytes, from_heap) \ 3165 do { \ 3166 (id) = (from_heap) ? \ 3167 sal_alloc((bytes), "ATP") : sal_dma_alloc((bytes), "ATP"); \ 3168 if ((id) == NULL) { \ 3169 atp_cleanup(); \ 3170 ATP_UNLOCK; \ 3171 return BCM_E_MEMORY; \ 3172 } \ 3173 sal_memset((void *)(id), 0, bytes); \ 3174 } while (0) 3175 3176 /* 3177 * Function: 3178 * _atp_base_init 3179 * Purpose: 3180 * Set up mutex and semaphores 3181 * Returns: 3182 * BCM_E_XXX 3183 * Notes: 3184 * Should be called before ATP_LOCK is taken 3185 */ 3186 3187 STATIC int 3188 _atp_base_init(void) 3189 { 3190 if (atp_tx_mutex == NULL) { 3191 atp_tx_mutex = sal_mutex_create("atp_tx_mutex"); 3192 if (atp_tx_mutex == NULL) { 3193 return BCM_E_MEMORY; 3194 } 3195 } 3196 if (atp_rx_mutex == NULL) { 3197 atp_rx_mutex = sal_mutex_create("atp_rx_mutex"); 3198 if (atp_rx_mutex == NULL) { 3199 return BCM_E_MEMORY; 3200 } 3201 } 3202 3203 base_init_done = TRUE; 3204 3205 return BCM_E_NONE; 3206 } 3207 3208 /* 3209 * Function: 3210 * _atp_base_init_check 3211 * Purpose: 3212 * Set up mutex and semaphores if needed 3213 * Returns: 3214 * BCM_E_XXX 3215 * Notes: 3216 * Should be called before ATP_LOCK is taken 3217 */ 3218 3219 STATIC int 3220 _atp_base_init_check(void) 3221 { 3222 BASE_INIT_CHECK; 3223 3224 return BCM_E_NONE; 3225 } 3226 3227 /* 3228 * Function: 3229 * _atp_init 3230 * Purpose: 3231 * Initialize the ATP subsystem 3232 * Returns: 3233 * BCM_E_XXX 3234 * Notes: 3235 * Pre-allocates TX transactions and sets up a free list. 3236 * Pre-allocates RX transactions and sets up a free list; installs 3237 * an ACK buffer into each RX transaction. 3238 */ 3239 3240 STATIC int 3241 _atp_init(void) 3242 { 3243 int i; 3244 int bytes; 3245 3246 BASE_INIT_CHECK; 3247 3248 ATP_LOCK; 3249 3250 /* Set up ATP RX transaction queue mutex */ 3251 if (atp_rxq_mutex == NULL) { 3252 atp_rxq_mutex = sal_mutex_create("atp_rxq"); 3253 if (atp_rxq_mutex == NULL) { 3254 ATP_UNLOCK; 3255 return BCM_E_MEMORY; 3256 } 3257 } 3258 3259 /* Set up BCM-RX queue and mutex */ 3260 if (atp_pkt_data_mutex == NULL) { 3261 atp_pkt_data_mutex = sal_mutex_create("atp_pkt_data"); 3262 if (atp_pkt_data_mutex == NULL) { 3263 ATP_UNLOCK; 3264 return BCM_E_MEMORY; 3265 } 3266 } 3267 3268 for (i = 0; i < ATP_PKT_DATA_QUEUE_LEN - 1; i++) { 3269 _atp_rx_pkt[i].next = &_atp_rx_pkt[i + 1]; 3270 } 3271 _atp_rx_pkt[ATP_PKT_DATA_QUEUE_LEN - 1].next = NULL; 3272 _atp_rcv_data_queue = NULL; 3273 _atp_rcv_data_queue_tail = NULL; 3274 _atp_pkt_data_freelist = &_atp_rx_pkt[0]; 3275 3276 if (atp_tx_sem == NULL) { 3277 atp_tx_sem = sal_sem_create("atp_tx_sem", sal_sem_BINARY, 0); 3278 if (atp_tx_sem == NULL) { 3279 ATP_UNLOCK; 3280 return BCM_E_MEMORY; 3281 } 3282 } 3283 3284 if (atp_rx_sem == NULL) { 3285 atp_rx_sem = sal_sem_create("atp_rx_sem", sal_sem_BINARY, 0); 3286 if (atp_rx_sem == NULL) { 3287 ATP_UNLOCK; 3288 return BCM_E_MEMORY; 3289 } 3290 } 3291 3292 /* Allocate the tx transactions and set up the free list */ 3293 if (tx_trans_pool != NULL) { 3294 sal_free((void *)tx_trans_pool); 3295 } 3296 bytes = sizeof(_atp_tx_trans_t) * atp_tx_pool_size; 3297 ALLOC_CHECK(tx_trans_pool, bytes, BCM_E_MEMORY); 3298 tx_trans_freelist = tx_trans_pool; 3299 3300 for (i = 0; i < atp_tx_pool_size; i++) { 3301 tx_trans_freelist[i].next = &tx_trans_freelist[i + 1]; 3302 } 3303 tx_trans_freelist[atp_tx_pool_size - 1].next = NULL; 3304 3305 /* Allocate and setup RX transactions and ACKs for each */ 3306 if (rx_trans_pool != NULL) { 3307 sal_free((void *)rx_trans_pool); 3308 } 3309 bytes = atp_rx_pool_size * sizeof(_atp_rx_trans_t); 3310 ALLOC_CHECK(rx_trans_pool, bytes, BCM_E_MEMORY); 3311 3312 bytes = _ATP_ACK_BYTES * atp_rx_pool_size; 3313 RX_ALLOC_CHECK(ack_pkt_data, bytes, atp_ack_pkt_data_from_heap); 3314 3315 rx_trans_freelist = rx_trans_pool; 3316 for (i = 0; i < atp_rx_pool_size; i++) { 3317 rx_trans_freelist[i].next = &rx_trans_freelist[i + 1]; 3318 rx_trans_freelist[i].ack_data = &ack_pkt_data[_ATP_ACK_BYTES * i]; 3319 } 3320 rx_trans_freelist[atp_rx_pool_size - 1].next = NULL; 3321 3322 init_done = TRUE; 3323 ATP_UNLOCK; 3324 3325 return BCM_E_NONE; 3326 } 3327 3328 /**************************************************************** 3329 * 3330 * ATP Callback Registration/De-registration 3331 */ 3332 3333 3334 /* 3335 * Function: 3336 * atp_client_add 3337 * Purpose: 3338 * Register a client ID for sending packets only 3339 * Parameters: 3340 * client_id - The client ID to register for 3341 * Returns: 3342 * BCM_E_XXX 3343 * Notes: 3344 * Just calls atp_register with trivial arguments. 3345 */ 3346 3347 int 3348 atp_client_add(int client_id) 3349 { 3350 return atp_register(client_id, 0, NULL, NULL, -1, -1); 3351 } 3352 3353 3354 /* 3355 * Function: 3356 * atp_register 3357 * Purpose: 3358 * Register a callback for an ATP client 3359 * Parameters: 3360 * client_id - The client ID to register for 3361 * flags - See atp.h. Indicates NH/C2C/ATP. 3362 * callback - Callback routine 3363 * cookie - Passed to callback 3364 * cos - The COS and internal priority 3365 * to use sending pkts for this client 3366 * If < 0, use default 3367 * (Unless overridden by flags in transmit) 3368 * vlan - The VLAN to use sending pkts for this client if valid 3369 * Returns: 3370 * BCM_E_XXX 3371 * Notes: 3372 * A client can register to send packets only (no RX) by registering 3373 * with a callback of NULL. 3374 * Will overwrite existing entry as long as flags agree. 3375 * 3376 * The 'cos' parameter contains the cos and internal priority 3377 * values encoded. Use CPUTRANS_COS_SET() CPUTRANS_INT_PRIO_SET() 3378 * to set values accordingly. The internal priority is optional; 3379 * if this is not provided, the cos value is used for internal priority. 3380 */ 3381 3382 int 3383 atp_register(int client_id, 3384 uint32 flags, 3385 atp_client_cb_f callback, 3386 void *cookie, 3387 int cos, 3388 int vlan) 3389 { 3390 _atp_client_t *client; 3391 3392 BASE_INIT_CHECK; /* Call init if that's not done yet */ 3393 3394 /* Create client if doesn't exist; 3395 * Fail if client exists with different flags */ 3396 ATP_LOCK; 3397 client = client_find(client_id); 3398 if (client == NULL) { 3399 client = client_id_add(client_id); 3400 if (client == NULL) { 3401 ATP_UNLOCK; 3402 return BCM_E_MEMORY; 3403 } 3404 } else { 3405 if (client->flags != flags) { 3406 ATP_UNLOCK; 3407 return BCM_E_EXISTS; 3408 } 3409 } 3410 3411 if (cos >= 0) { 3412 client->cos = CPUTRANS_COS_GET(cos); 3413 if (cos & CPUTRANS_INT_PRIO_VALID) { 3414 CPUTRANS_INT_PRIO_SET(client->cos, CPUTRANS_INT_PRIO_GET(cos)); 3415 } 3416 } 3417 3418 if (vlan >= 0 && vlan < 4096) { 3419 client->vlan = vlan; 3420 } 3421 3422 client->flags = flags; 3423 client->callback = callback; 3424 client->cookie = cookie; 3425 3426 ATP_UNLOCK; 3427 3428 return BCM_E_NONE; 3429 } 3430 3431 3432 /* 3433 * Function: 3434 * atp_unregister 3435 * Purpose: 3436 * Unregister a callback for an ATP client registered with atp_register 3437 * Parameters: 3438 * client_id - Client ID to unregister 3439 * Returns: 3440 * BCM_E_XXX 3441 */ 3442 3443 void 3444 atp_unregister(int client_id) 3445 { 3446 _atp_client_t *cli; 3447 3448 if (!init_done) { 3449 return; 3450 } 3451 3452 ATP_LOCK; 3453 cli = client_find(client_id); 3454 if (cli != NULL) { 3455 client_delete(cli, TRUE); 3456 } 3457 ATP_UNLOCK; 3458 } 3459 3460 3461 /**************************************************************** 3462 * 3463 * ATP Transmit Support 3464 */ 3465 3466 /* 3467 * If synchronous ATP operation, wait on semaphor. 3468 * 3469 * The return code is stored via a pointer put into the 3470 * transaction. This routine is responsible for deleting 3471 * the transaction. 3472 * 3473 * A timeout is detected by retransmit attempts. 3474 */ 3475 3476 STATIC int 3477 _atp_sync_check(_atp_tx_trans_t *trans, atp_tx_cb_f callback, int dest_cpu) 3478 { 3479 int rv = BCM_E_NONE; 3480 3481 if (TX_TRANS_ACK(trans) && (callback == NULL)) { 3482 ++tx_sleep_count; 3483 if (sal_sem_take(trans->tx_sem, sal_sem_FOREVER) < 0) { 3484 rv = BCM_E_INTERNAL; 3485 } else { 3486 rv = trans->tx_rv; 3487 } 3488 ATP_TX_LOCK; 3489 atp_tx_trans_delete(trans); 3490 ATP_TX_UNLOCK; 3491 --tx_sleep_count; 3492 } 3493 3494 return rv; 3495 } 3496 3497 /**************************************************************** 3498 * 3499 * Send out a TX transaction. 3500 * For BET, just send out everything and return. 3501 * For ATP, check next expected byte from RX and start with 3502 * that segment. 3503 */ 3504 3505 STATIC void 3506 _atp_tx_trans_send(_atp_tx_trans_t *trans) 3507 { 3508 bcm_pkt_t *first_pkt; /* In list to send out */ 3509 int rv = BCM_E_NONE; 3510 c2c_cb_f c2c_cb = NULL; 3511 next_hop_tx_callback_f nh_cb = NULL; 3512 int i; 3513 int next_hop; 3514 int no_ack; 3515 3516 next_hop = TX_TRANS_NEXT_HOP(trans); 3517 no_ack = TX_TRANS_NO_ACK(trans); 3518 3519 first_pkt = trans->pkt_list; 3520 assert(first_pkt->next != first_pkt); 3521 if (!no_ack) { /* Use ACK */ 3522 LOG_DEBUG(BSL_LS_TKS_ATP, 3523 (BSL_META("TX ATP send cli %d, seq %d txcount %d lasttx %u\n"), 3524 trans->client->client_id, 3525 trans->_atp_hdr.seq_num, 3526 trans->tx_count, trans->last_tx)); 3527 /* ATP packet; check bytes acked; always send sync */ 3528 for (i = 1; trans->bytes_acked >= (i * trans->seg_len); i++) { 3529 first_pkt = first_pkt->next; 3530 } 3531 ATP_ASSERT((first_pkt != NULL)); 3532 c2c_cb = _atp_c2c_tx_callback; 3533 nh_cb = _atp_nh_tx_callback; 3534 } 3535 trans->flags |= _ATP_TX_F_PENDING; 3536 3537 if (next_hop) { 3538 if (trans->tx_count > 0) { /* Update with new seq num, etc */ 3539 next_hop_pkt_update(first_pkt, ATP_PKT_TYPE, 3540 CPU_KEY(trans->dest_cpu)); 3541 } 3542 rv = next_hop_pkt_send(first_pkt, NULL, NULL); /* Always sync */ 3543 if (nh_cb != NULL) { 3544 nh_cb(rv, NULL, (void *)trans); 3545 } 3546 } else { /* Send C2C */ 3547 int update; /* Protect from update changing in midstream */ 3548 update = atp_db_update_count; 3549 if (update - trans->db_update > 0) { 3550 /* DB has been updated; freshen packet */ 3551 if (c2c_pkt_update(first_pkt, CPU_KEY(trans->dest_cpu)) >= 0) { 3552 /* Success, indicate db updated for transaction */ 3553 trans->db_update = update; 3554 } 3555 } 3556 3557 rv = c2c_pkt_send(first_pkt, NULL, NULL); /* Always sync */ 3558 if (c2c_cb != NULL) { 3559 c2c_cb(NULL, (void *)trans); 3560 } 3561 } 3562 3563 if (rv != BCM_E_NONE) { 3564 trans->flags |= _ATP_TX_ERROR_SEEN; 3565 LOG_ERROR(BSL_LS_TKS_ATP, 3566 (BSL_META("ATP TX error %d: %s\n"), 3567 rv, bcm_errmsg(rv))); 3568 } else { 3569 if (!no_ack) { 3570 if (trans->tx_count++ > 0) { 3571 INCR_COUNTER(tx_retry_cnt); 3572 } 3573 trans->last_tx = sal_time_usecs(); 3574 } 3575 } 3576 } 3577 3578 /* Allocate and store a sequence number for a transaction */ 3579 3580 STATIC void 3581 _atp_seq_num_set(int no_ack, _atp_tx_trans_t *trans) 3582 { 3583 bcm_pkt_t *cur_pkt; 3584 uint16 seq_num; 3585 3586 TX_SEQ_NUM_GET(no_ack, seq_num, trans->client, trans->dest_cpu); 3587 trans->_atp_hdr.seq_num = seq_num; 3588 /* Pack the sequence number for each segment */ 3589 for (cur_pkt = trans->pkt_list; cur_pkt != NULL; cur_pkt = cur_pkt->next) { 3590 ATP_SEQ_NUM_SET(cur_pkt->pkt_data[0].data, seq_num); 3591 } 3592 } 3593 3594 /* 3595 * Transaction is an ATP request. If sync, create a semaphor 3596 * to wait on; enqueue the packet. 3597 */ 3598 3599 3600 /* Set up ATP or BET transaction according to client flags */ 3601 STATIC int 3602 _atp_tx_trans_setup(_atp_tx_trans_t *trans, int *trans_deleted) 3603 { 3604 atp_tx_cb_f callback; 3605 3606 callback = trans->callback; 3607 trans->next = NULL; 3608 3609 if (TX_TRANS_NO_ACK(trans)) { /* Best effort */ 3610 if (callback == NULL) { /* Send out immediately */ 3611 _atp_seq_num_set(TRUE, trans); 3612 _atp_tx_trans_send(trans); 3613 atp_tx_trans_delete(trans); 3614 *trans_deleted = TRUE; 3615 } else { /* Place in BET queue */ 3616 _atp_seq_num_set(TRUE, trans); 3617 trans->prev = bet_queue_tail; 3618 if (bet_queue_tail != NULL) { 3619 bet_queue_tail->next = trans; 3620 } else { 3621 bet_queue = trans; 3622 } 3623 bet_queue_tail = trans; 3624 trans->flags |= _ATP_TX_F_ENQUEUED; 3625 *trans_deleted = FALSE; 3626 } 3627 } else { /* ACK required */ 3628 _atp_client_t *client; 3629 int dest_cpu; 3630 3631 client = trans->client; 3632 dest_cpu = trans->dest_cpu; 3633 3634 if (callback == NULL) { /* Sync send; create semaphor */ 3635 trans->tx_sem = sal_sem_create("atp_tx", sal_sem_BINARY, 0); 3636 if (trans->tx_sem == NULL) { 3637 atp_tx_trans_delete(trans); 3638 *trans_deleted = TRUE; 3639 LOG_ERROR(BSL_LS_TKS_ATP, 3640 (BSL_META("ATP TX: Failed to create sem\n"))); 3641 return BCM_E_MEMORY; 3642 } 3643 trans->flags |= _ATP_TX_F_SEM_WAITING; 3644 } 3645 3646 _atp_seq_num_set(FALSE, trans); 3647 /* Enqueue transaction */ 3648 trans->prev = client->cpu[dest_cpu].tx_tail; 3649 if (client->cpu[dest_cpu].tx_tail != NULL) { 3650 client->cpu[dest_cpu].tx_tail->next = trans; 3651 } else { /* Queue was empty; add to front */ 3652 client->cpu[dest_cpu].tx_trans = trans; 3653 } 3654 client->cpu[dest_cpu].tx_tail = trans; 3655 trans->flags |= _ATP_TX_F_ENQUEUED; 3656 ++atp_tx_pending; 3657 *trans_deleted = FALSE; 3658 } 3659 3660 return BCM_E_NONE; 3661 } 3662 3663 /* 3664 * Send a packet without ACK. No checking is done here. It is assumed 3665 * that the CPUTRANS header is allocated at the beginning of the packet, 3666 * and that the packet fits in a single segment. 3667 * 3668 * Note: Transactions are used here in a bad way. They are used to 3669 * convey the callback and cookie to the async transport call. 3670 * In those, they are freed directly back to the transaction free list 3671 * rather than being marked deleted. 3672 * 3673 * CPUTRANS_COS_OVERRIDE and CPUTRANS_INT_PRIO_OVERRIDE in ct_flags 3674 * override default cos and internal priority values. 3675 */ 3676 3677 STATIC int 3678 _atp_simple_send(cpudb_key_t dest_key, 3679 _atp_client_t *client, 3680 uint8 *pkt_buf, 3681 int len, 3682 uint32 ct_flags, 3683 atp_tx_cb_f callback, 3684 void *cookie) 3685 { 3686 _atp_tx_trans_t *trans = NULL; 3687 _atp_hdr_t _atp_hdr; 3688 int rv = BCM_E_NONE; 3689 int next_hop; 3690 int cos; 3691 3692 next_hop = (client->flags & ATP_F_NEXT_HOP) || 3693 (ct_flags & CPUTRANS_NEXT_HOP); 3694 3695 /* Default COS comes from client; flags may override */ 3696 cos = client->cos; 3697 if (ct_flags & CPUTRANS_COS_OVERRIDE) { 3698 CPUTRANS_COS_SET(cos, CPUTRANS_COS_GET(ct_flags)); 3699 } else { 3700 /* To ensure proper TX pkt allocation */ 3701 CPUTRANS_COS_SET(ct_flags, cos); /* Only cos is needed */ 3702 } 3703 3704 /* If internal priority is provided in flag, override default value */ 3705 if (ct_flags & CPUTRANS_INT_PRIO_OVERRIDE) { 3706 CPUTRANS_INT_PRIO_SET(cos, CPUTRANS_INT_PRIO_GET(ct_flags)); 3707 } 3708 3709 _atp_hdr.client_id = client->client_id; 3710 TX_BET_SEQ_NUM_GET(_atp_hdr.seq_num, client); 3711 _atp_hdr.tot_bytes = len - CPUTRANS_HEADER_BYTES; 3712 _atp_hdr.start_byte = 0; 3713 _atp_hdr.tot_segs = 1; 3714 _atp_hdr.opcode = _ATP_OPC_DATA; 3715 _atp_hdr.segment = 0; 3716 _atp_hdr.hdr_flags = _ATP_HDR_NO_ACK; 3717 _atp_hdr.cos = CPUTRANS_COS_GET(cos); /* Only cos */ 3718 if (next_hop) { 3719 _atp_hdr.hdr_flags |= _ATP_HDR_NEXT_HOP; 3720 } 3721 _atp_hdr_pack(pkt_buf, &_atp_hdr); 3722 3723 if (callback != NULL) { /* Async */ 3724 /* For aync, use a TX trans to pass back cookie and callback */ 3725 trans = tx_trans_freelist; 3726 if (trans == NULL) { /* Allocation failed */ 3727 return BCM_E_RESOURCE; 3728 } 3729 tx_trans_freelist = trans->next; 3730 INCR_COUNTER(txraw_grab); 3731 3732 sal_memset((void *)trans, 0, sizeof(_atp_tx_trans_t)); 3733 trans->callback = callback; 3734 trans->cookie = cookie; 3735 } 3736 3737 LOG_VERBOSE(BSL_LS_TKS_TX, 3738 (BSL_META("ATP simple %d: NH %d. ctf %x. cb %p\n"), 3739 client->client_id, next_hop, ct_flags, callback)); 3740 if (next_hop) { /* Next hop packet */ 3741 rv = next_hop_tx(pkt_buf, 3742 len, 3743 cos, 3744 client->vlan, 3745 _atp_seg_len, 3746 ct_flags, 3747 ATP_PKT_TYPE, 3748 dest_key, 3749 callback == NULL ? NULL : bet_nh_free_tx_cb, 3750 (void *)trans); 3751 } else { /* c2c directed packet */ 3752 rv = c2c_tx(dest_key, pkt_buf, len, 3753 cos, client->vlan, _atp_seg_len, 3754 ATP_PKT_TYPE, ct_flags, 3755 callback == NULL ? NULL : bet_c2c_free_tx_cb, 3756 (void *)trans); 3757 } 3758 3759 return rv; 3760 } 3761 3762 /**************************************************************** 3763 * 3764 * Simple BET support and TX Callbacks 3765 */ 3766 3767 /* This is registered with c2c transmit routine when sending async */ 3768 STATIC void 3769 _atp_c2c_tx_callback(uint8 *pkt_buf, void *cookie) 3770 { 3771 _atp_tx_trans_t *trans; 3772 3773 COMPILER_REFERENCE(pkt_buf); 3774 3775 trans = (_atp_tx_trans_t *)cookie; 3776 3777 3778 trans->flags &= ~_ATP_TX_F_PENDING; 3779 } 3780 3781 /* This is registered with nexthop transmit routine when sending async */ 3782 STATIC void 3783 _atp_nh_tx_callback(int rv, uint8 *pkt_buf, void *cookie) 3784 { 3785 _atp_tx_trans_t *trans; 3786 3787 COMPILER_REFERENCE(pkt_buf); 3788 3789 trans = (_atp_tx_trans_t *)cookie; 3790 3791 3792 trans->tx_rv = rv; 3793 trans->flags &= ~_ATP_TX_F_PENDING; 3794 } 3795 3796 /**************************************************************** 3797 * BET single segment fast track support; only called by _atp_simple_send 3798 */ 3799 3800 /* 3801 * Async callback which releases transaction; these are 3802 * for best-effort only, so no semaphor in TX transaction. 3803 */ 3804 3805 STATIC void 3806 _tx_free_callback(_atp_tx_trans_t *trans, uint8 *pkt_buf, int rv) 3807 { 3808 atp_tx_cb_f cb; 3809 3810 cb = trans->callback; 3811 3812 if (cb != NULL) { 3813 cb(pkt_buf, trans->cookie, rv); 3814 } 3815 3816 ATP_TX_LOCK; 3817 atp_tx_trans_delete(trans); 3818 ATP_TX_UNLOCK; 3819 } 3820 3821 /* Next hop async callback for simple BET */ 3822 STATIC void 3823 bet_nh_free_tx_cb(int rv, uint8 *pkt_buf, void *cookie) 3824 { 3825 _tx_free_callback((_atp_tx_trans_t *)cookie, pkt_buf, rv); 3826 } 3827 3828 /* C2C async callback for simple BET */ 3829 STATIC void 3830 bet_c2c_free_tx_cb(uint8 *pkt_buf, void *cookie) 3831 { 3832 _tx_free_callback((_atp_tx_trans_t *)cookie, pkt_buf, BCM_E_NONE); 3833 } 3834 3835 /**************************************************************** 3836 * 3837 * ATP TX Thread 3838 */ 3839 3840 /* Assumes lock held */ 3841 3842 STATIC void 3843 tx_done_handle(_atp_client_t *client, int cpu, _atp_tx_trans_t *trans) 3844 { 3845 3846 if (trans->flags & _ATP_TX_F_TIMEOUT) { 3847 LOG_ERROR(BSL_LS_TKS_ATP, 3848 (BSL_META("ATP: TX timeout, seq %d. " CPUDB_KEY_FMT 3849 " cli %d. to %d tx cnt %d.\n"), 3850 trans->_atp_hdr.seq_num, 3851 CPUDB_KEY_DISP(_atp_cpu_info[trans->dest_cpu].key), 3852 trans->client->client_id, 3853 trans->dest_cpu, 3854 trans->tx_count)); 3855 if (atp_timeout_cb != NULL) { 3856 (*atp_timeout_cb)(CPU_KEY(trans->dest_cpu)); 3857 } 3858 } 3859 3860 if (trans->callback != NULL) { 3861 trans->callback(trans->pkt_buf, trans->cookie, trans->tx_rv); 3862 } 3863 atp_tx_trans_delete(trans); 3864 } 3865 3866 STATIC void 3867 _set_retrx_flag(_atp_tx_trans_t *trans) 3868 { 3869 uint8 *ptr; 3870 3871 if (!(trans->_atp_hdr.hdr_flags & _ATP_HDR_RETRANSMIT)) { 3872 trans->_atp_hdr.hdr_flags |= _ATP_HDR_RETRANSMIT; 3873 3874 ptr = ATP_HEADER_START(trans->pkt_list->pkt_data[0].data); 3875 ptr += (sizeof(uint16) + sizeof(uint16)); /* version + cli ID */ 3876 PACK_LONG(ptr, trans->_atp_hdr.hdr_flags); 3877 } 3878 } 3879 3880 /* Return boolean, TRUE means transaction is active */ 3881 STATIC int 3882 tx_trans_retransmit_check(_atp_tx_trans_t *trans) 3883 { 3884 if (trans->flags & _ATP_TX_F_DONE) { 3885 return FALSE; /* Not active */ 3886 } 3887 3888 if (trans->last_tx == 0) { /* Never sent before */ 3889 _atp_tx_trans_send(trans); 3890 } else { 3891 int dt = SAL_USECS_SUB(sal_time_usecs(), trans->last_tx); 3892 3893 if (dt < 0 || dt > atp_retry_timeout) { 3894 /* Set retransmit flag in ATP header */ 3895 _set_retrx_flag(trans); 3896 3897 /* Time to retransmit this data; check for too many retries */ 3898 if (trans->tx_count > atp_retry_count) { 3899 trans->tx_rv = BCM_E_TIMEOUT; 3900 trans->flags |= _ATP_TX_F_DONE | _ATP_TX_F_TIMEOUT; 3901 INCR_COUNTER(tx_timeout_cnt); 3902 return FALSE; /* No long active. */ 3903 } else { 3904 if (!(trans->flags & (_ATP_TX_F_PENDING | _ATP_TX_F_DONE))) { 3905 _atp_tx_trans_send(trans); 3906 } 3907 } 3908 } 3909 } 3910 3911 /* This is the active transmit transaction */ 3912 return TRUE; 3913 } 3914 3915 /* 3916 * Go through all clients and check the state of any pending transmit 3917 * transactions. At most one is active for any given client/dest CPU. 3918 * It may require retransmitting. 3919 * 3920 * Only send out data for the first active (non-ack'd) transaction. 3921 */ 3922 3923 STATIC void 3924 tx_transactions_check(void) 3925 { 3926 int cpu; 3927 int idx; 3928 _atp_client_t *client; 3929 _atp_tx_trans_t *trans; 3930 _atp_tx_trans_t *trans_next; 3931 int active_found = FALSE; 3932 3933 ATP_TX_LOCK; 3934 FOREACH_CLIENT(client, idx) { 3935 for (cpu = 0; cpu < atp_cpu_max; cpu++) { 3936 active_found = FALSE; 3937 trans = client->cpu[cpu].tx_trans; 3938 while (trans != NULL) { 3939 /* Need to store next transaction now, in case current 3940 transaction is deleted from list */ 3941 trans_next = trans->next; 3942 3943 if (!active_found) { 3944 /* At most one transaction per client/dest CPU may 3945 be active at a time */ 3946 active_found = tx_trans_retransmit_check(trans); 3947 } 3948 if (trans->flags & _ATP_TX_F_DONE && 3949 !(trans->flags & _ATP_TX_F_PENDING)) { 3950 LOG_DEBUG(BSL_LS_TKS_ATP, 3951 (BSL_META("TX done cli %d, to %d, seq %d\n"), 3952 client->client_id, cpu, 3953 trans->_atp_hdr.seq_num)); 3954 tx_done_handle(client, cpu, trans); 3955 } 3956 3957 trans = trans_next; 3958 } 3959 } 3960 } 3961 ATP_TX_UNLOCK; 3962 } 3963 3964 /* This is max BET queue entries processed before returning */ 3965 3966 #ifndef MAX_BET_COUNT 3967 #define MAX_BET_COUNT 4 3968 #endif 3969 3970 3971 STATIC void 3972 bet_tx_queue_handle(void) 3973 { 3974 _atp_tx_trans_t *entry; 3975 int bet_count = 0; 3976 3977 if (atp_tx_thread_exit) { /* Exit forced */ 3978 return; 3979 } 3980 3981 ATP_TX_LOCK; 3982 while (bet_queue != NULL) { 3983 if (bet_count++ >= MAX_BET_COUNT) { 3984 break; 3985 } 3986 entry = bet_queue; 3987 bet_queue = bet_queue->next; 3988 if (bet_queue == NULL) { 3989 bet_queue_tail = NULL; 3990 } 3991 3992 _atp_tx_trans_send(entry); 3993 if (entry->callback != NULL) { 3994 entry->callback(entry->pkt_buf, 3995 entry->cookie, BCM_E_NONE); 3996 } 3997 atp_tx_trans_delete(entry); 3998 } 3999 ATP_TX_UNLOCK; 4000 } 4001 4002 STATIC int 4003 _handle_tx_data(_atp_pkt_data_t *pkt_p) 4004 { 4005 cpudb_key_t src_key; 4006 int src_cpu; 4007 uint8 *pkt_buf; 4008 _atp_hdr_t _atp_hdr; 4009 4010 pkt_buf = pkt_p->pkt_buf; 4011 _atp_hdr_unpack(pkt_buf, &_atp_hdr); 4012 4013 /* Look for the source CPU */ 4014 CPUDB_KEY_UNPACK(&pkt_buf[CPUTRANS_SRC_KEY_OFS], src_key); 4015 src_cpu = _atp_key_lookup(src_key); 4016 if (src_cpu < 0) { 4017 LOG_VERBOSE(BSL_LS_TKS_ATP, 4018 (BSL_META("ATP ACK pkt: could not find source CPU key\n"))); 4019 return BCM_RX_HANDLED; 4020 } 4021 4022 return atp_ack_handle(src_cpu, &_atp_hdr); 4023 } 4024 4025 /* Process transmit related packet queue */ 4026 4027 STATIC void 4028 trx_process_pkt_data(void) 4029 { 4030 _atp_pkt_data_t *cur_p; 4031 _atp_pkt_data_t *next_p; 4032 int rv; 4033 4034 ATP_TX_LOCK; 4035 ATP_PKT_DATA_LOCK; /* Steal the current queue of pkts */ 4036 cur_p = _atp_trx_data_queue; 4037 _atp_trx_data_queue_tail = NULL; 4038 _atp_trx_data_queue = NULL; 4039 ATP_PKT_DATA_UNLOCK; 4040 4041 while (cur_p != NULL) { 4042 next_p = cur_p->next; 4043 4044 rv = _handle_tx_data(cur_p); 4045 if (rv != BCM_RX_HANDLED_OWNED) { /* Free packet data */ 4046 _atp_trans_ptr->tp_data_free(_atp_trans_ptr->tp_unit, 4047 cur_p->pkt_buf); 4048 } 4049 ATP_PKT_DATA_LOCK; /* Free the pkt struct */ 4050 cur_p->next = _atp_pkt_data_freelist; 4051 _atp_pkt_data_freelist = cur_p; 4052 ATP_PKT_DATA_UNLOCK; 4053 4054 cur_p = next_p; 4055 } 4056 ATP_TX_UNLOCK; 4057 } 4058 4059 STATIC void 4060 atp_tx_thread(void *cookie) 4061 { 4062 int cur_timeout; 4063 4064 COMPILER_REFERENCE(cookie); 4065 4066 LOG_VERBOSE(BSL_LS_TKS_ATP, 4067 (BSL_META("ATP: TX Thread starting\n"))); 4068 atp_tx_thread_exit = FALSE; 4069 while (1) { 4070 cur_timeout = atp_tx_pending ? atp_retry_timeout : ATP_LONG_TIMEOUT; 4071 if (bet_queue == NULL) { /* Don't sleep if best effort transactions */ 4072 sal_sem_take(atp_tx_sem, cur_timeout); 4073 } 4074 4075 if (atp_tx_thread_exit) { /* Exit forced */ 4076 break; 4077 } 4078 4079 if (_atp_trx_data_queue != NULL) { 4080 trx_process_pkt_data(); 4081 } 4082 4083 tx_transactions_check(); 4084 bet_tx_queue_handle(); 4085 } 4086 4087 /* Give any pending TX transactions */ 4088 atp_tx_thread_id = SAL_THREAD_ERROR; 4089 LOG_VERBOSE(BSL_LS_TKS_ATP, 4090 (BSL_META("ATP: TX Thread exiting\n"))); 4091 sal_thread_exit(0); 4092 } 4093 4094 /* 4095 * Macro to check if this is a single segment, no-ACK packet with 4096 * the transport header already allocated. 4097 */ 4098 4099 #define _ATP_SIMPLE_CHECK(_no_ack, _ct_flags, _len) \ 4100 ((_no_ack) && ((_ct_flags) & CPUTRANS_NO_HEADER_ALLOC) && \ 4101 ((_len) - CPUTRANS_HEADER_BYTES <= _atp_seg_len)) 4102 4103 /* 4104 * Function: 4105 * atp_tx 4106 * Purpose: 4107 * Send out a packet to given CPU using ATP; override may occur per CPU 4108 * Parameters: 4109 * dest_key - Dest CPU to send to 4110 * client_id - Client ID to send to 4111 * pkt_buf - Pointer to data to transmit 4112 * len - Length of packet from pkt_buf to end, excluding CRC 4113 * unless CPUTRANS_CRC_REGEN is set 4114 * ct_flags - Bitmap of flags passed in 4115 * CPUTRANS_NO_HEADER_ALLOC Packet contains space for headers 4116 * CPUTRANS_BROADCAST Send to all CPUs; 4117 * Currently only supported on NO ACK, next hop. 4118 * CPUTRANS_COS_OVERRIDE Override default cos 4119 * CPUTRANS_INT_PRIO_OVERRIDE Overrude default internal priority 4120 * CPUTRANS_CRC_REGEN Caller must allocate space for Ethernet 4121 * transport CRC 4122 * callback - If not NULL, do async and callback 4123 * cookie - Passed on callback 4124 * Returns: 4125 * BCM_E_XXX 4126 * Notes: 4127 * The client must have been registered. 4128 * 4129 * Depending on the flags (CPUTRANS_NO_HEADER_ALLOC), the 4130 * pkt_buf pointer may either point to the beginning of the 4131 * payload or the beginning of the actual packet. 4132 * 4133 * When callback is specified, it will take place after 4134 * the packet has been acknowledged. 4135 * 4136 * The application is responsible for freeing the data sent. If 4137 * the data are sent asynchronously then the application must not 4138 * deallocate the data until the callback is made indicating the 4139 * transmit is complete. 4140 * 4141 * atp_tx creates a BCM packet via cputrans_tx_pkt_list_alloc 4142 * and frees that structure on completion of the transaction. 4143 */ 4144 4145 int 4146 atp_tx(cpudb_key_t dest_key, 4147 int client_id, 4148 uint8 *pkt_buf, 4149 int len, 4150 uint32 ct_flags, 4151 atp_tx_cb_f callback, 4152 void *cookie) 4153 { 4154 _atp_tx_trans_t *trans; 4155 int rv = BCM_E_NONE; 4156 _atp_client_t *client; 4157 int dest_cpu; 4158 int no_ack = FALSE; /* If true, packet is best effort */ 4159 atp_tx_f callout; 4160 int trans_deleted; 4161 4162 LOG_DEBUG(BSL_LS_TKS_ATP, 4163 (BSL_META("ATP tx cli %d%s\n"), 4164 client_id, 4165 _atp_running ? "" : " (not running)")); 4166 4167 ATP_INIT_CHECK; 4168 4169 if (ct_flags & CPUTRANS_CRC_REGEN) { 4170 ct_flags &= ~CPUTRANS_CRC_REGEN; 4171 len -= sizeof(uint32); /* accept flag, but ignore */ 4172 } 4173 4174 /* Check for override of TX function (before "running" check) */ 4175 ATP_TX_LOCK; 4176 dest_cpu = _atp_key_lookup(dest_key); 4177 if (dest_cpu >= 0) { /* Found dest CPU */ 4178 if (_atp_cpu_info[dest_cpu].override_tx != NULL) { /* Override TX */ 4179 callout = _atp_cpu_info[dest_cpu].override_tx; 4180 ATP_TX_UNLOCK; 4181 LOG_DEBUG(BSL_LS_TKS_ATP, 4182 (BSL_META("ATP tx calling override %d\n"), 4183 dest_cpu)); 4184 return callout(dest_key, client_id, pkt_buf, len, ct_flags, 4185 callback, cookie); 4186 } 4187 if (_atp_cpu_info[dest_cpu].flags & _ATP_CPU_NO_ACK) { 4188 no_ack = TRUE; /* CPU forces NO-ACK */ 4189 } 4190 } 4191 4192 if (!_atp_running) { 4193 ATP_TX_UNLOCK; 4194 INCR_COUNTER(atp_not_running); 4195 return BCM_E_INIT; 4196 } 4197 4198 client = client_find(client_id); 4199 if (client == NULL) { /* Client must exist */ 4200 ATP_TX_UNLOCK; 4201 INCR_COUNTER(invalid_client_cnt); 4202 LOG_VERBOSE(BSL_LS_TKS_ATP, 4203 (BSL_META("ATP TX: Client not found\n"))); 4204 return BCM_E_NOT_FOUND; 4205 } 4206 4207 LOG_DEBUG(BSL_LS_TKS_ATP, 4208 (BSL_META("TX cli %d, flags 0x%x ctflags 0x%x, cb %p to " 4209 CPUDB_KEY_FMT_EOLN), 4210 client_id, client->flags, ct_flags, callback, 4211 CPUDB_KEY_DISP(dest_key))); 4212 4213 if (ct_flags & CPUTRANS_BCAST) { /* Forces no-ACK and next hop */ 4214 no_ack = TRUE; 4215 } else { /* Not a broadcast pkt */ 4216 if (IS_LOCAL_CPU_KEY(dest_key)) { /* Packet is directed loopback */ 4217 rv = _atp_loopback(client, dest_cpu, pkt_buf, len, ct_flags); 4218 ATP_TX_UNLOCK; 4219 if(BCM_FAILURE(rv)) { 4220 INCR_COUNTER(lb_pkt_send_fail); 4221 } 4222 4223 /* All done with operation */ 4224 if (callback != NULL) { 4225 callback(pkt_buf, cookie, rv); 4226 } 4227 return rv; 4228 } 4229 if (dest_cpu < 0) { 4230 ATP_TX_UNLOCK; 4231 INCR_COUNTER(invalid_dest_cpu_cnt); 4232 return BCM_E_NOT_FOUND; 4233 } 4234 4235 if (CLI_IS_BET(client) || (ct_flags & CPUTRANS_NO_ACK)) { 4236 /* Client is BET or packet forces no-ack */ 4237 no_ack = TRUE; 4238 } 4239 4240 if (_ATP_SIMPLE_CHECK(no_ack, ct_flags, len)) { 4241 /* Okay, it's a simple operation, send it out */ 4242 rv = _atp_simple_send(dest_key, client, pkt_buf, 4243 len, ct_flags, callback, cookie); 4244 ATP_TX_UNLOCK; 4245 if (BCM_FAILURE(rv)) { 4246 INCR_COUNTER(tx_simple_send_fail); 4247 } 4248 return rv; 4249 } 4250 } 4251 4252 /* Create and setup the transaction */ 4253 trans = _atp_tx_trans_create(dest_cpu, client, no_ack, pkt_buf, 4254 len, ct_flags, callback, cookie); 4255 4256 if (trans == NULL) { 4257 ATP_TX_UNLOCK; 4258 LOG_VERBOSE(BSL_LS_TKS_ATP, 4259 (BSL_META("TX unable to alloc trans/pkt, cli %d\n"), 4260 client_id)); 4261 return BCM_E_RESOURCE; 4262 } 4263 4264 rv = _atp_tx_trans_setup(trans, &trans_deleted); 4265 4266 ATP_TX_UNLOCK; 4267 if (rv == BCM_E_NONE) { 4268 ATP_TX_THREAD_WAKE; 4269 /* If ATP sync send, wait on semaphor for completion */ 4270 if (!trans_deleted) { 4271 rv = _atp_sync_check(trans, callback, dest_cpu); 4272 if (BCM_FAILURE(rv)) { 4273 INCR_COUNTER(tx_send_fail); 4274 } 4275 } 4276 } else { 4277 INCR_COUNTER(tx_trans_setup_fail); 4278 } 4279 4280 return rv; 4281 } 4282 4283 /**************************************************************** 4284 * 4285 * Data and packet free routines 4286 */ 4287 4288 /* 4289 * Function: 4290 * atp_rx_data_alloc 4291 * Purpose: 4292 * Allocate a buffer from the ATP transport memory pool 4293 * Parameters: 4294 * bytes - Number of bytes to be allocated for payload 4295 * Returns: 4296 * Pointer to payload buffer if successful 4297 * NULL if failure 4298 * Notes: 4299 * Buffer allocated must be freed with 'atp_rx_free()'. 4300 * Application must free the buffer ONLY if the called 4301 * routine (to which the buffer is given) return code 4302 * is NOT BCM_RX_HANDLED_OWNED. 4303 */ 4304 4305 void * 4306 atp_rx_data_alloc(int bytes) 4307 { 4308 uint8 *buffer = NULL; 4309 uint8 *data_ptr = NULL; 4310 4311 /* Allocate space from transport memory pool */ 4312 4313 _atp_trans_ptr->tp_data_alloc(_atp_trans_ptr->tp_unit, 4314 bytes + CPUTRANS_HEADER_BYTES, 4315 0, (void*)&buffer); 4316 if (buffer != NULL) { 4317 data_ptr = buffer + CPUTRANS_HEADER_BYTES; 4318 } else { 4319 INCR_COUNTER(rx_data_alloc_fail); 4320 } 4321 4322 return (void *)data_ptr; 4323 } 4324 4325 4326 /* 4327 * Function: 4328 * atp_rx_free 4329 * Purpose: 4330 * Free buffers that are stolen by atp callbacks 4331 * Parameters: 4332 * payload_ptr - Payload pointer passed to callback 4333 * pkt_ptr - Pointer to BCM packet struct passed to callback 4334 * Returns: 4335 * void 4336 * Notes: 4337 * When a callback returns BCM_RX_HANDLED_OWNED, the packet 4338 * data has been "stolen" by the application. In order to free 4339 * this data, the application MUST call atp_rx_free 4340 * on the payload pointer and packet pointer passed to the callback. 4341 */ 4342 4343 void 4344 atp_rx_free(void *payload_ptr, void *pkt_ptr) 4345 { 4346 bcm_pkt_t *pkt = (bcm_pkt_t *)pkt_ptr; 4347 uint8 *data = (uint8 *)payload_ptr; 4348 4349 if (pkt == NULL) { 4350 /* It's a generic RX pointer, no packet to free */ 4351 if (data != NULL) { 4352 _atp_trans_ptr->tp_data_free(_atp_trans_ptr->tp_unit, data); 4353 } 4354 } else { 4355 _atp_rx_pkt_free(pkt); 4356 } 4357 } 4358 4359 /* 4360 * Function: 4361 * atp_tx_data_alloc 4362 * Purpose: 4363 * Allocate buffer from the ATP transport memory pool 4364 * Parameters: 4365 * bytes - Number of bytes to be allocated for payload 4366 * Returns: 4367 * Pointer to payload buffer if successful 4368 * NULL if failure 4369 * Notes: 4370 * Buffer allocated must be freed with 'atp_tx_data_free()'. 4371 */ 4372 4373 void * 4374 atp_tx_data_alloc(int bytes) 4375 { 4376 uint8 *buffer = NULL; 4377 4378 /* Allocate space from transport memory pool */ 4379 4380 _atp_trans_ptr->tp_data_alloc(_atp_trans_ptr->tp_unit, bytes, 4381 0, (void*)&buffer); 4382 if (NULL == buffer) { 4383 INCR_COUNTER(tx_data_alloc_fail); 4384 } 4385 return (void *)buffer; 4386 } 4387 4388 4389 /* 4390 * Function: 4391 * atp_tx_data_free 4392 * Purpose: 4393 * Free buffer allocated by atp_tx_data_alloc 4394 * Parameters: 4395 * buffer - buffer to free 4396 * Returns: 4397 * void 4398 * Notes: 4399 * Unlike atp_rx_free, the buffer pointer must be exactly 4400 * the pointer returned by atp_tx_data_alloc(). 4401 */ 4402 4403 void 4404 atp_tx_data_free(void *buffer) 4405 { 4406 _atp_trans_ptr->tp_data_free(_atp_trans_ptr->tp_unit, buffer); 4407 } 4408 4409 /**************************************************************** 4410 * 4411 * ATP Packet Receive functions 4412 */ 4413 4414 4415 /* 4416 * Function: 4417 * atp_cpudb_keys_add 4418 * Purpose: 4419 * Add all CPU keys in a CPU DB to those known by ATP 4420 * Parameters: 4421 * db_ref - Reference to CPU DB to scan 4422 * Returns: 4423 * BCM_E_XXX 4424 * Notes: 4425 * It's not an error if entries already exist 4426 */ 4427 4428 int 4429 atp_cpudb_keys_add(cpudb_ref_t db_ref) 4430 { 4431 cpudb_entry_t *entry; 4432 4433 /* Scan the DB for new addresses */ 4434 CPUDB_FOREACH_ENTRY(db_ref, entry) { 4435 BCM_IF_ERROR_RETURN(atp_key_add(entry->base.key, 4436 entry->flags & CPUDB_F_IS_LOCAL)); 4437 } 4438 4439 return BCM_E_NONE; 4440 } 4441 4442 4443 /* 4444 * Function: 4445 * atp_key_add 4446 * Purpose: 4447 * Add a CPU key to those known to ATP 4448 * Parameters: 4449 * key - CPUDB key of CPU to add 4450 * is_local - Update local record if set 4451 * Returns: 4452 * The key index if successful; else error. 4453 * Notes: 4454 * It's not an error if the entry already exists 4455 */ 4456 4457 int 4458 atp_key_add(cpudb_key_t key, int is_local) 4459 { 4460 int idx; 4461 int rv; 4462 4463 ATP_INIT_CHECK; 4464 4465 ATP_LOCK; 4466 if (is_local) { 4467 sal_memcpy(&_atp_local_key, &key, sizeof(cpudb_key_t)); 4468 } 4469 idx = _atp_key_lookup(key); 4470 if (idx >= 0) { 4471 ATP_UNLOCK; 4472 return BCM_E_NONE; /* Already exists, no error */ 4473 } 4474 rv = _atp_key_add(key); 4475 ATP_UNLOCK; 4476 4477 return rv < 0 ? BCM_E_RESOURCE : BCM_E_NONE; 4478 } 4479 4480 /* Callback to purge queues on detach */ 4481 void 4482 atp_attach_callback(int unit, int attach, cpudb_entry_t *cpuent, int cpuunit) 4483 { 4484 if (attach) { 4485 LOG_VERBOSE(BSL_LS_TKS_ATP, 4486 (BSL_META_U(unit, 4487 "ATP attach unit %d; key " CPUDB_KEY_FMT_EOLN), 4488 unit, CPUDB_KEY_DISP(cpuent->base.key))); 4489 atp_key_add(cpuent->base.key, cpuent->flags & CPUDB_F_IS_LOCAL); 4490 } else { 4491 if (!(cpuent->flags & CPUDB_F_IS_LOCAL)) { 4492 LOG_VERBOSE(BSL_LS_TKS_ATP, 4493 (BSL_META_U(unit, 4494 "ATP detach unit %d; key " CPUDB_KEY_FMT_EOLN), 4495 unit, CPUDB_KEY_DISP(cpuent->base.key))); 4496 atp_key_remove(cpuent->base.key); 4497 next_hop_key_invalidate(cpuent->base.key); 4498 } 4499 } 4500 } 4501 4502 /* 4503 * Function: 4504 * atp_key_purge 4505 * Purpose: 4506 * Purge all the current TX and RX transactions associated with a cpu 4507 * Parameters: 4508 * key - CPUDB key of CPU to purge 4509 * Returns: 4510 * BCM_E_XXX; 4511 * Notes: 4512 * ATP must be initialized. 4513 */ 4514 4515 int 4516 atp_key_purge(cpudb_key_t key) 4517 { 4518 int rv = BCM_E_NONE; 4519 int idx; 4520 4521 if (!init_done) { 4522 return BCM_E_INIT; 4523 } 4524 4525 ATP_LOCK; 4526 idx = _atp_key_lookup(key); 4527 if (idx < 0) { 4528 rv = BCM_E_NOT_FOUND; 4529 } else { 4530 _atp_cpu_purge(idx); 4531 } 4532 ATP_UNLOCK; 4533 4534 return rv; 4535 } 4536 4537 /* 4538 * Function: 4539 * atp_cpu_remove 4540 * Purpose: 4541 * Remove a CPU from those known to ATP 4542 * Parameters: 4543 * key - CPUDB key of CPU to remove 4544 * Returns: 4545 * BCM_E_NONE 4546 * Notes: 4547 * It's not an error if the entry does not exist, but ATP must 4548 * be initialized. 4549 */ 4550 4551 int 4552 atp_key_remove(cpudb_key_t key) 4553 { 4554 int idx; 4555 int rv = BCM_E_NONE; 4556 4557 if (!init_done) { 4558 return BCM_E_INIT; 4559 } 4560 4561 ATP_LOCK; 4562 idx = _atp_key_lookup(key); 4563 if (idx < 0) { 4564 rv = BCM_E_NONE; /* Not there, no error */ 4565 } else { 4566 _atp_cpu_remove(idx); 4567 } 4568 ATP_UNLOCK; 4569 4570 return rv; 4571 } 4572 4573 4574 /* 4575 * Function: 4576 * atp_cpu_remove_all 4577 * Purpose: 4578 * Remove all CPUs from ATP 4579 * Returns: 4580 * BCM_E_NONE 4581 */ 4582 4583 int 4584 atp_cpu_remove_all(void) 4585 { 4586 int idx; 4587 4588 ATP_INIT_CHECK; 4589 ATP_LOCK; 4590 for (idx = 0; idx < CPUDB_CPU_MAX; idx++) { 4591 if (_atp_cpu_info[idx].flags & _ATP_CPU_VALID) { 4592 _atp_cpu_remove(idx); 4593 } 4594 } 4595 ATP_UNLOCK; 4596 4597 return BCM_E_NONE; 4598 } 4599 4600 /* 4601 * Function: 4602 * atp_cpu_count_set 4603 * Purpose: 4604 * DEPRECATED: Set the max number of CPUs that ATP supports 4605 * Parameters: 4606 * count - Number of CPUs to support 4607 * Returns: 4608 * BCM_E_XXX 4609 * Notes: 4610 * No longer a configurable parameter 4611 */ 4612 4613 int 4614 atp_cpu_count_set(int count) 4615 { 4616 if (!init_done) { 4617 return BCM_E_INIT; 4618 } 4619 4620 if (count > CPUDB_CPU_MAX) { 4621 return BCM_E_RESOURCE; 4622 } 4623 4624 return BCM_E_NONE; 4625 } 4626 4627 /* 4628 * Function: 4629 * atp_cpu_count_get 4630 * Purpose: 4631 * DEPRECATED: Get the number of CPUs supported by ATP 4632 * Returns: 4633 * Number of CPUs supported by ATP 4634 */ 4635 4636 int 4637 atp_cpu_count_get(void) 4638 { 4639 return CPUDB_CPU_MAX; 4640 } 4641 4642 4643 /* 4644 * Function: 4645 * atp_tx_override_set/get 4646 * Purpose: 4647 * Set (get) the atp_tx override function for the given dest CPU key 4648 * Parameters: 4649 * dest_cpu - Which CPU entry to update 4650 * override_tx - The function pointer to use (get: OUTPUT) 4651 * Returns: 4652 * BCM_E_XXX 4653 * Notes: 4654 * On set, will add the CPU key to the local records if 4655 * not found. 4656 */ 4657 4658 int 4659 atp_tx_override_set(cpudb_key_t dest_cpu, atp_tx_f override_tx) 4660 { 4661 int idx; 4662 4663 ATP_INIT_CHECK; 4664 /* See if CPU exists; if not add it */ 4665 ATP_LOCK; 4666 idx = _atp_key_lookup(dest_cpu); 4667 if (idx < 0) { 4668 idx = _atp_key_add(dest_cpu); 4669 } 4670 if (idx >= 0) { 4671 _atp_cpu_info[idx].override_tx = override_tx; 4672 } 4673 ATP_UNLOCK; 4674 4675 return idx < 0 ? BCM_E_FAIL : BCM_E_NONE; 4676 } 4677 4678 int 4679 atp_tx_override_get(cpudb_key_t dest_cpu, atp_tx_f *override_tx) 4680 { 4681 int idx; 4682 int rv = BCM_E_NONE; 4683 4684 if (override_tx == NULL) { 4685 return BCM_E_PARAM; 4686 } 4687 4688 ATP_INIT_CHECK; 4689 4690 /* See if CPU exists; if not add it */ 4691 ATP_LOCK; 4692 idx = _atp_key_lookup(dest_cpu); 4693 if (idx < 0) { 4694 *override_tx = NULL; 4695 rv = BCM_E_NOT_FOUND; 4696 } else { 4697 *override_tx = _atp_cpu_info[idx].override_tx; 4698 } 4699 ATP_UNLOCK; 4700 4701 return rv; 4702 } 4703 4704 4705 /* 4706 * Function: 4707 * atp_tx_override_set/get 4708 * Purpose: 4709 * Set (get) the atp_tx override function for the given dest CPU key 4710 * Parameters: 4711 * dest_cpu - Which CPU entry to update 4712 * override_tx - The function pointer to use (get: OUTPUT) 4713 * Returns: 4714 * BCM_E_XXX 4715 * Notes: 4716 * On set, will add the CPU key to the local records if 4717 * not found. 4718 */ 4719 4720 int 4721 atp_cpu_no_ack_set(cpudb_key_t dest_cpu, int no_ack) 4722 { 4723 int idx; 4724 4725 ATP_INIT_CHECK; 4726 /* See if CPU exists; if not add it */ 4727 ATP_LOCK; 4728 idx = _atp_key_lookup(dest_cpu); 4729 if (idx < 0) { 4730 idx = _atp_key_add(dest_cpu); 4731 } 4732 if (idx >= 0) { 4733 if (no_ack) { 4734 _atp_cpu_info[idx].flags |= _ATP_CPU_NO_ACK; 4735 } else { 4736 _atp_cpu_info[idx].flags &= ~_ATP_CPU_NO_ACK; 4737 } 4738 } 4739 ATP_UNLOCK; 4740 4741 return idx < 0 ? BCM_E_FAIL : BCM_E_NONE; 4742 } 4743 4744 int 4745 atp_cpu_no_ack_get(cpudb_key_t dest_cpu, int *no_ack) 4746 { 4747 int idx; 4748 int rv = BCM_E_NONE; 4749 4750 if (no_ack == NULL) { 4751 return BCM_E_PARAM; 4752 } 4753 4754 ATP_INIT_CHECK; 4755 4756 /* See if CPU exists; if not add it */ 4757 ATP_LOCK; 4758 idx = _atp_key_lookup(dest_cpu); 4759 if (idx < 0) { 4760 rv = BCM_E_NOT_FOUND; 4761 } else { 4762 *no_ack = (_atp_cpu_info[idx].flags & _ATP_CPU_NO_ACK) != 0; 4763 } 4764 ATP_UNLOCK; 4765 4766 return rv; 4767 } 4768 4769 4770 /* 4771 * Function: 4772 * atp_rx_inject 4773 * Purpose: 4774 * Inject a packet into the ATP callback sequence 4775 * Parameters: 4776 * src_key - From whence arrived 4777 * client_id - Who to give the packet to 4778 * pkt_buf - Pointer to data 4779 * len - Length of data in bytes 4780 * Returns: 4781 * BCM RX return type indicating if packet data is stolen. 4782 * Notes: 4783 * Generally used in conjunction with overriding atp_tx. See 4784 * comments at top of file. 4785 * 4786 * This always passes NULL for the packet pointer to the callback. 4787 */ 4788 4789 bcm_rx_t 4790 atp_rx_inject(cpudb_key_t src_key, int client_id, uint8 *pkt_buf, int len) 4791 { 4792 _atp_client_t *client; 4793 atp_client_cb_f cb; 4794 void *cookie; 4795 bcm_rx_t rv = BCM_RX_HANDLED; 4796 4797 if (BCM_FAILURE(_atp_base_init_check())) { 4798 return BCM_RX_HANDLED; 4799 } 4800 4801 ATP_RX_LOCK; 4802 client = client_find(client_id); 4803 if (client == NULL) { 4804 LOG_WARN(BSL_LS_TKS_ATP, 4805 (BSL_META("ATP rx inject: Unknown client id %d\n"), 4806 client_id)); 4807 ATP_RX_UNLOCK; 4808 INCR_COUNTER(invalid_client_cnt); 4809 return BCM_RX_HANDLED; 4810 } 4811 4812 if (client->callback != NULL) { 4813 cb = client->callback; 4814 cookie = client->cookie; 4815 ATP_RX_UNLOCK; 4816 rv = cb(src_key, client_id, NULL, pkt_buf, len, cookie); 4817 } else { 4818 ATP_RX_UNLOCK; 4819 } 4820 4821 return rv; 4822 } 4823 4824 /* 4825 * Function: 4826 * atp_db_update_notify 4827 * Purpose: 4828 * Atomically increment 'atp_db_update_count'. 4829 * It will be used to inform the ATP send pasth that 4830 * the DB entry has changed so refresh the pkt hdr. 4831 * Returns: 4832 * none 4833 */ 4834 void 4835 atp_db_update_notify(void) 4836 { 4837 ATP_LOCK; 4838 atp_db_update_count++; 4839 ATP_UNLOCK; 4840 } 4841 4842 #if defined(BROADCOM_DEBUG) 4843 4844 void 4845 atp_counter_dump(void) 4846 { 4847 LOG_CLI((BSL_META("tx_retry_cnt = %d\n"), 4848 tx_retry_cnt)); 4849 LOG_CLI((BSL_META("tx_timeout_cnt = %d\n"), 4850 tx_timeout_cnt)); 4851 LOG_CLI((BSL_META("reassem_alloc_fail = %d\n"), 4852 reassem_alloc_fail)); 4853 LOG_CLI((BSL_META("rx_trans_fail = %d\n"), 4854 rx_trans_fail)); 4855 LOG_CLI((BSL_META("stale_rx_trans = %d\n"), 4856 stale_rx_trans)); 4857 LOG_CLI((BSL_META("rx_pkt_drops = %d\n"), 4858 rx_pkt_drops)); 4859 LOG_CLI((BSL_META("ack_pkt_drops = %d\n"), 4860 ack_pkt_drops)); 4861 LOG_CLI((BSL_META("stale_rx_trans = %d\n"), 4862 stale_rx_trans)); 4863 LOG_CLI((BSL_META("rxt_pkt_alloc_fail = %d\n"), 4864 rxt_pkt_alloc_fail)); 4865 LOG_CLI((BSL_META("rx_mseg_alloc_fail = %d\n"), 4866 rx_mseg_alloc_fail)); 4867 LOG_CLI((BSL_META("tx_trans_fail = %d\n"), 4868 tx_trans_fail)); 4869 LOG_CLI((BSL_META("txt_pkt_alloc_fail = %d\n"), 4870 txt_pkt_alloc_fail)); 4871 LOG_CLI((BSL_META("lb_buf_alloc_fail = %d\n"), 4872 lb_buf_alloc_fail)); 4873 LOG_CLI((BSL_META("gc_deferrals = %d\n"), 4874 gc_deferrals)); 4875 LOG_CLI((BSL_META("gc_blocked = %d\n"), 4876 gc_blocked)); 4877 LOG_CLI((BSL_META("cli_del_tx_busy = %d\n"), 4878 cli_del_tx_busy)); 4879 LOG_CLI((BSL_META("clients_deleted = %d\n"), 4880 clients_deleted)); 4881 LOG_CLI((BSL_META("tx_data_alloc_fail = %d\n"), 4882 tx_data_alloc_fail)); 4883 LOG_CLI((BSL_META("rx_data_alloc_fail = %d\n"), 4884 rx_data_alloc_fail)); 4885 LOG_CLI((BSL_META("invalid_client_cnt = %d\n"), 4886 invalid_client_cnt)); 4887 LOG_CLI((BSL_META("invalid_dest_cpu_cnt = %d\n"), 4888 invalid_dest_cpu_cnt)); 4889 LOG_CLI((BSL_META("lb_pkt_send_fail = %d\n"), 4890 lb_pkt_send_fail)); 4891 LOG_CLI((BSL_META("tx_simple_send_fail = %d\n"), 4892 tx_simple_send_fail)); 4893 LOG_CLI((BSL_META("tx_trans_setup_fail = %d\n"), 4894 tx_trans_setup_fail)); 4895 LOG_CLI((BSL_META("tx_send_fail = %d\n"), 4896 tx_send_fail)); 4897 LOG_CLI((BSL_META("atp_not_running = %d\n"), 4898 atp_not_running)); 4899 } 4900 4901 void 4902 atp_dump(int verbose) 4903 { 4904 int i, cpu; 4905 _atp_tx_trans_t *tx_trans; 4906 _atp_rx_trans_t *rx_trans; 4907 _atp_client_t *client; 4908 int cos, int_prio; 4909 4910 LOG_CLI((BSL_META("Init %d. run %d.\n"), 4911 init_done, _atp_running)); 4912 if (verbose) { 4913 for (i = 0; i < CPUDB_CPU_MAX; i++) { 4914 if (CPU_VALID(i)) { 4915 LOG_CLI((BSL_META(" CPU %d " CPUDB_KEY_FMT_EOLN), 4916 i, CPUDB_KEY_DISP(CPU_KEY(i)))); 4917 } 4918 } 4919 } 4920 4921 LOG_CLI((BSL_META("atp_tx_mutex %p\n"), 4922 atp_tx_mutex)); 4923 LOG_CLI((BSL_META("atp_rx_mutex %p\n"), 4924 atp_rx_mutex)); 4925 LOG_CLI((BSL_META("atp_rxq_mutex %p\n"), 4926 atp_rxq_mutex)); 4927 LOG_CLI((BSL_META("atp_tx_sem: %p\n"), 4928 atp_tx_sem)); 4929 LOG_CLI((BSL_META("atp_rx_sem: %p\n"), 4930 atp_rx_sem)); 4931 LOG_CLI((BSL_META("tx_pending %d. sleep cnt %d.\n"), 4932 atp_tx_pending, tx_sleep_count)); 4933 LOG_CLI((BSL_META("Cntrs: rxt_cr %d. rxt_free %d. rx raw free %d.\n"), 4934 rxt_create, rxt_free, rxraw_free)); 4935 LOG_CLI((BSL_META(" txt_cr %d\n"), 4936 txt_create)); 4937 LOG_CLI((BSL_META(" rx raw grab %d. tx raw grab %d.\n"), 4938 rxraw_grab, txraw_grab)); 4939 LOG_CLI((BSL_META("Drops: bet %d. atp %d. mem %d. slf %d. " 4940 "old rx %d.\n"), 4941 bet_rx_drop, atp_rx_drop, mem_rx_drop, slf_rx_drop, 4942 old_rx_trans_drop)); 4943 4944 LOG_CLI((BSL_META("tx free %p. rx free %p.\n"), 4945 tx_trans_freelist, 4946 rx_trans_freelist)); 4947 4948 if (verbose) { /* Show counters */ 4949 atp_counter_dump(); 4950 } 4951 4952 FOREACH_CLIENT(client, i) { 4953 cos = CPUTRANS_COS_GET(client->cos); 4954 if (client->cos & CPUTRANS_INT_PRIO_VALID) { 4955 int_prio = CPUTRANS_INT_PRIO_GET(client->cos); 4956 } else { 4957 int_prio = cos; 4958 } 4959 4960 LOG_CLI((BSL_META("Client %d. fl 0x%x. cos %d int_prio %d. vl %d. bet sn %d\n"), 4961 client->client_id, client->flags, cos, int_prio, 4962 client->vlan, client->bet_tx_seq_num)); 4963 if (verbose) { 4964 for (cpu = 0; cpu < CPUDB_CPU_MAX ; cpu++) { 4965 if (client->cpu[cpu].cpu_flags != 0) { 4966 LOG_CLI((BSL_META(" CPU %d: Flags 0x%x. rx SN %d. tx SN %d.\n"), 4967 cpu, client->cpu[cpu].cpu_flags, 4968 client->cpu[cpu].rx_seq_num, 4969 client->cpu[cpu].tx_seq_num)); 4970 } 4971 4972 for (rx_trans = client->cpu[cpu].rx_trans; 4973 rx_trans != NULL; rx_trans = rx_trans->next) { 4974 LOG_CLI((BSL_META(" CPU %d: RX %p: flags 0x%x. cpu %d. rsegs %d. " 4975 "ack %d.\n"), cpu, rx_trans, rx_trans->flags, 4976 rx_trans->src_cpu, rx_trans->rcv_segs, 4977 rx_trans->ack_count)); 4978 LOG_CLI((BSL_META(" time %u. cli %d. pkt %p. len %d. seq %d\n"), 4979 rx_trans->rcvd_time, rx_trans->client->client_id, 4980 rx_trans->pkt, rx_trans->payload_len, 4981 rx_trans->_atp_hdr.seq_num)); 4982 } 4983 4984 for (tx_trans = client->cpu[cpu].tx_trans; 4985 tx_trans != NULL; tx_trans = tx_trans->next) { 4986 LOG_CLI((BSL_META(" CPU %d: TX %p: flags 0x%x. ct_flags 0x%x. " 4987 "cpu %d. len %d.\n"), cpu, tx_trans, 4988 tx_trans->flags, tx_trans->ct_flags, 4989 tx_trans->dest_cpu, tx_trans->len)); 4990 LOG_CLI((BSL_META(" b ack %d. last tx %u. tx_rv %d. " 4991 "cli %d. sem %p\n"), tx_trans->bytes_acked, 4992 tx_trans->last_tx, tx_trans->tx_rv, 4993 tx_trans->client->client_id, 4994 tx_trans->tx_sem)); 4995 } 4996 } 4997 } 4998 } 4999 5000 LOG_CLI((BSL_META("BET queue %p tail %p\n"), 5001 bet_queue, bet_queue_tail)); 5002 LOG_CLI((BSL_META("atp_cpu_max: %d\n"), 5003 atp_cpu_max)); 5004 } 5005 5006 #endif /* BROADCOM_DEBUG */