rpc.c (31856B)
1 /* 2 * 3 * This license is set out in https://raw.githubusercontent.com/Broadcom-Network-Switching-Software/OpenBCM/master/Legal/LICENSE file. 4 * 5 * Copyright 2007-2019 Broadcom Inc. All rights reserved. 6 * 7 * BCM Dispatch Remote Procedure Call Utilities 8 */ 9 10 #include <shared/bsl.h> 11 12 #include <sdk_config.h> 13 #include <shared/alloc.h> 14 #include <sal/core/libc.h> 15 #include <sal/core/sync.h> 16 #include <sal/core/thread.h> 17 #include <sal/appl/sal.h> 18 #include <soc/drv.h> 19 20 #include <shared/idents.h> 21 22 #include <bcm/types.h> 23 #include <bcm/error.h> 24 #include <bcm/init.h> 25 26 #include <appl/cpudb/cpudb.h> 27 #include <appl/cputrans/atp.h> 28 29 #include <bcm_int/control.h> 30 #include <bcm_int/rpc/pack.h> 31 #include <bcm_int/rpc/rpc.h> 32 #include <bcm_int/rpc/server.h> 33 #include <bcm_int/client_dispatch.h> 34 #include <soc/cmdebug.h> 35 36 #ifdef BCM_RPC_SUPPORT 37 #define RPC_SERVER_MIN_THREADS 1 38 39 #define RPC_CLIENT_ID SHARED_CLIENT_ID_RPC 40 41 #ifndef RPC_THREAD_STACK 42 #define RPC_THREAD_STACK SAL_THREAD_STKSZ 43 #endif 44 45 #ifndef RPC_THREAD_PRIO 46 #define RPC_THREAD_PRIO 50 47 #endif 48 49 #ifndef RPC_REPLY_TIMEOUT 50 #define RPC_REPLY_TIMEOUT sal_sem_FOREVER 51 #endif 52 53 #define RPC_SLEEP(thrdid) sal_sem_take(_rpc_server_sems [thrdid], sal_sem_FOREVER) 54 #define RPC_WAKE(thrdid) sal_sem_give(_rpc_server_sems [thrdid]) 55 #define RPC_SLOCK(thrdid) sal_mutex_take(_rpc_server_locks [thrdid], sal_mutex_FOREVER) 56 #define RPC_SUNLOCK(thrdid) sal_mutex_give(_rpc_server_locks [thrdid]) 57 #define RPC_CLOCK sal_mutex_take(_rpc_client_lock, sal_mutex_FOREVER) 58 #define RPC_CUNLOCK sal_mutex_give(_rpc_client_lock) 59 60 #define _RPC_SERVER_NAME_BASE "bcm_rpc_server_" /*cannot be more than 50 char long*/ 61 #define _RPC_SERVER_NAME_EXT "00" 62 #define _RPC_SERVER_NAME _RPC_SERVER_NAME_BASE _RPC_SERVER_NAME_EXT 63 #define _RPC_SERVER_NAME_EXT_PTR sizeof (_RPC_SERVER_NAME_BASE) 64 65 /* 66 * RPC Server Thread Availablity Database Operations 67 */ 68 #define BCM_RPC_STHREAD_INIT (1 << 0) 69 #define BCM_RPC_STHREAD_SET (1 << 1) 70 #define BCM_RPC_STHREAD_GET (1 << 2) 71 #define BCM_RPC_STHREAD_NEXT (1 << 3) 72 73 /* 74 * RPC Server Thread Availablity Status Options 75 */ 76 #define BCM_RPC_SERVER_THREAD_USED (1 << 0) 77 #define BCM_RPC_SERVER_THREAD_FREE (1 << 1) 78 79 typedef struct _rpc_sthread_db_s { 80 uint32 thread_id; 81 uint32 status; 82 }_rpc_sthread_db_t; 83 84 /* client rpc request: stored on the requesting thread's stack */ 85 typedef struct _rpc_creq_s { 86 struct _rpc_creq_s *next; 87 sal_sem_t sem; 88 int unit; 89 uint32 seq; 90 uint8 *rbuf; 91 void *cookie; 92 #ifdef BROADCOM_DEBUG 93 sal_usecs_t time; 94 #endif /* BROADCOM_DEBUG */ 95 } _rpc_creq_t; 96 97 static sal_mutex_t _rpc_client_lock; 98 static uint32 _rpc_seq; 99 static _rpc_creq_t *_rpc_creq = NULL; 100 101 static sal_mutex_t *_rpc_server_locks; /* each thread has its own lock */ 102 static sal_sem_t *_rpc_server_sems; /* each thread has its own semaphore */ 103 static sal_thread_t _rpc_server_thread = SAL_THREAD_ERROR; 104 105 static int *_rpc_thread_exit; /* each thread has its own exit flag */ 106 static bcm_rpc_sreq_t **_rpc_sreqs; /* each thread has its own queue */ 107 static bcm_rpc_sreq_t **_rpc_sreqs_tail; /* each thread has its own queue */ 108 109 static _rpc_sthread_db_t *_rpc_s_db; /* each thread has its own free 110 * database 111 */ 112 static int _num_rpc_threads = 113 RPC_SERVER_MIN_THREADS; /* this variable defines how many 114 * rpc threads to start 115 */ 116 static int _rpc_next_thread = 0; /* using round-robin, once all 117 * available free threads are 118 * used */ 119 int _rpc_nexthop = 0; 120 121 #ifdef BROADCOM_DEBUG 122 #define RPC_COUNT(_c) ++(_c) 123 static volatile int _rpc_count_c_request; 124 static volatile int _rpc_count_c_reply; 125 static volatile int _rpc_count_c_fail; 126 static volatile int _rpc_count_c_timeout; 127 static volatile int _rpc_count_c_noreq; 128 static volatile int _rpc_count_c_detach; 129 static volatile int _rpc_count_s_request; 130 static volatile int _rpc_count_s_reply; 131 static volatile int _rpc_count_s_rretry; 132 static volatile int _rpc_count_s_rerr; 133 static volatile int _rpc_count_s_merr; 134 static volatile int _rpc_count_s_run; 135 static volatile int _rpc_count_s_wrongver; 136 static volatile int _rpc_count_s_nokey; 137 #else 138 #define RPC_COUNT(_c) 139 #endif /* BROADCOM_DEBUG */ 140 141 /* 142 * Allocate DMAable memory suitable for an RPC packet. 143 * Fill in an RPC header with appropriate arguments. 144 * This memory is freed in bcm_rpc_request or bcm_rpc_reply. 145 */ 146 uint8 * 147 bcm_rpc_setup(uint8 dir, uint32 *key, uint32 len, uint32 seq, uint32 arg) 148 { 149 uint8 *pkt, *pp; 150 int i; 151 152 pkt = atp_tx_data_alloc(BCM_RPC_HLEN+len); 153 if (pkt == NULL) { 154 return NULL; 155 } 156 157 pp = pkt; 158 BCM_PACK_U32(pp, seq); 159 BCM_PACK_U8(pp, dir); 160 BCM_PACK_U8(pp, BCM_RPC_VERSION); 161 BCM_PACK_U16(pp, BCM_RPC_HLEN+len); 162 for (i = 0; i < BCM_RPC_LOOKUP_KEYLEN; i++) { 163 BCM_PACK_U32(pp, key ? key[i] : 0); 164 } 165 166 /* 167 * arg is: 168 * unit for client requests 169 * return value for server responses 170 */ 171 BCM_PACK_U32(pp, arg); 172 assert((pp - pkt) <= (BCM_RPC_HLEN + len)); 173 return pkt; 174 } 175 176 /* 177 * Free the payload of a received packet 178 */ 179 void 180 bcm_rpc_free(uint8 *buf, void *cookie) 181 { 182 atp_rx_free((void *)buf, cookie); 183 } 184 185 /* 186 * Unlink a request 187 */ 188 STATIC void 189 _bcm_rpc_unlink_request(_rpc_creq_t *req) 190 { 191 _rpc_creq_t *preq; 192 193 /* take req out of _rpc_creq */ 194 RPC_CLOCK; 195 if (_rpc_creq == req) { 196 _rpc_creq = req->next; 197 } else { 198 for (preq = _rpc_creq; preq; preq = preq->next) { 199 if (preq->next == req) { 200 preq->next = req->next; 201 break; 202 } 203 } 204 } 205 RPC_CUNLOCK; 206 } 207 208 #ifdef BCM_RPC_ATP_TX_CALLBACK 209 /* 210 * The ATP works more efficiently if a callback function is used to free 211 * the buffer. If a callback function is not used then ATP creates and 212 * destroys a semaphore for each packet. 213 */ 214 STATIC void 215 bcm_rpc_atp_tx_callback (uint8 *pkt, void *cookie, int rv) 216 { 217 atp_tx_data_free(pkt); 218 } 219 #endif 220 221 /* 222 * Send a request, wait the a reply 223 */ 224 int 225 bcm_rpc_request(int unit, uint8 *buf, int len, uint8 **rbuf, 226 void **cookie) 227 { 228 int rv; 229 uint32 seq; 230 uint8 *bp; 231 cpudb_key_t cpu; 232 _rpc_creq_t *req; 233 atp_tx_cb_f atp_tx_cb = NULL; 234 235 if (_rpc_client_lock == NULL) { 236 atp_tx_data_free(buf); 237 return BCM_E_UNAVAIL; 238 } 239 240 RPC_COUNT(_rpc_count_c_request); 241 242 req = (_rpc_creq_t *)sal_alloc(sizeof(_rpc_creq_t), "bcm_rpc_client_req"); 243 if (req == NULL) { 244 atp_tx_data_free(buf); 245 return BCM_E_MEMORY; 246 } 247 memset(req, 0, (sizeof(_rpc_creq_t))); 248 249 req->sem = sal_sem_create("bcm_rpc_send", sal_sem_BINARY, 0); 250 if (req->sem == NULL) { 251 RPC_COUNT(_rpc_count_c_fail); 252 atp_tx_data_free(buf); 253 sal_free((void *)req); 254 return BCM_E_MEMORY; 255 } 256 req->unit = unit; 257 req->rbuf = NULL; 258 #ifdef BROADCOM_DEBUG 259 req->time = sal_time_usecs(); 260 #endif /* BROADCOM_DEBUG */ 261 262 /* link req on stack into _rpc_creq list */ 263 RPC_CLOCK; 264 seq = ++_rpc_seq; 265 req->seq = seq; 266 req->next = _rpc_creq; 267 _rpc_creq = req; 268 RPC_CUNLOCK; 269 270 bp = buf; 271 BCM_PACK_U32(bp, seq); 272 273 if (!BCM_UNIT_VALID(unit) || (BCM_CONTROL(unit)->drv_control == NULL)) { 274 _bcm_rpc_unlink_request(req); 275 sal_sem_destroy(req->sem); 276 atp_tx_data_free(buf); 277 sal_free((void *)req); 278 return BCM_E_MEMORY; 279 } 280 281 cpu = *(cpudb_key_t *)BCM_CONTROL(unit)->drv_control; 282 #ifdef BCM_RPC_ATP_TX_CALLBACK 283 /* 284 * Set atp_tx_cb for async 285 * This callback is responsible to free the packet buffer 286 */ 287 atp_tx_cb = bcm_rpc_atp_tx_callback; 288 #endif 289 rv = atp_tx(cpu, RPC_CLIENT_ID, buf, len, 0, atp_tx_cb, NULL); 290 #ifdef BCM_RPC_ATP_TX_CALLBACK 291 if (rv < 0) 292 #endif 293 { 294 atp_tx_data_free(buf); 295 } 296 if (rv < 0) { 297 /* take req out of _rpc_creq on error */ 298 _bcm_rpc_unlink_request(req); 299 sal_sem_destroy(req->sem); 300 sal_free((void *)req); 301 RPC_COUNT(_rpc_count_c_fail); 302 return rv; 303 } 304 305 /* wait for reply */ 306 sal_sem_take(req->sem, RPC_REPLY_TIMEOUT); 307 308 if (req->rbuf == NULL) { /* no response buffer */ 309 /* This request normally would have been unlinked when req.sem 310 was given, but if sal_sem_take returns in error then the 311 request record needs to be be unlinked here. 312 */ 313 _bcm_rpc_unlink_request(req); 314 sal_sem_destroy(req->sem); 315 sal_free((void *)req); 316 RPC_COUNT(_rpc_count_c_timeout); 317 return BCM_E_TIMEOUT; 318 } 319 sal_sem_destroy(req->sem); 320 *rbuf = req->rbuf; 321 *cookie = req->cookie; 322 sal_free((void *)req); 323 324 return BCM_E_NONE; 325 } 326 327 #ifndef BCM_RPC_REPLY_RETRY_COUNT 328 #define BCM_RPC_REPLY_RETRY_COUNT 3 329 #endif 330 331 #ifndef BCM_RPC_REPLY_RETRY_DELAY 332 #define BCM_RPC_REPLY_RETRY_DELAY 10000 333 #endif 334 335 336 337 338 /* 339 * Send a reply 340 */ 341 int 342 bcm_rpc_reply(cpudb_key_t cpu, uint8 *buf, int len) 343 { 344 int rv = BCM_E_INTERNAL; 345 int retry = BCM_RPC_REPLY_RETRY_COUNT; 346 atp_tx_cb_f atp_tx_cb = NULL; 347 348 if (buf == NULL) { 349 RPC_COUNT(_rpc_count_s_merr); 350 return BCM_E_MEMORY; 351 } 352 #ifdef BCM_RPC_ATP_TX_CALLBACK 353 /* 354 * Set atp_tx_cb for async 355 * This callback is responsible to free the packet buffer 356 */ 357 atp_tx_cb = bcm_rpc_atp_tx_callback; 358 #endif 359 RPC_COUNT(_rpc_count_s_reply); 360 while (retry) { 361 rv = atp_tx(cpu, RPC_CLIENT_ID, buf, len, 0, atp_tx_cb, NULL); 362 if (rv != BCM_E_RESOURCE) { 363 break; 364 } else { 365 RPC_COUNT(_rpc_count_s_rretry); 366 /* Retry atp_tx() on resource errors before giving up */ 367 retry--; 368 sal_usleep(BCM_RPC_REPLY_RETRY_DELAY); 369 } 370 } 371 #ifdef BCM_RPC_ATP_TX_CALLBACK 372 if (rv < 0) 373 #endif 374 { 375 atp_tx_data_free(buf); 376 } 377 #ifdef BROADCOM_DEBUG 378 if (BCM_FAILURE(rv)) { 379 RPC_COUNT(_rpc_count_s_rerr); 380 } 381 #endif /* BROADCOM_DEBUG */ 382 383 return rv; 384 } 385 386 /* 387 * Fail any old version 1 requests with a generic BCM_E_UNAVAIL error 388 */ 389 #define BCM_RPC_V1_HLEN 14 390 STATIC void 391 _bcm_rpc_unavail_v1(cpudb_key_t r_cpu, uint8 *r_pkt, void *r_cookie) 392 { 393 uint32 seq; 394 uint8 dir, ver; 395 uint32 sig; 396 uint16 entry; 397 uint8 *pp; 398 399 /* decode the v1 RPC header */ 400 pp = r_pkt; 401 BCM_UNPACK_U32(pp, seq); 402 BCM_UNPACK_U8(pp, dir); 403 BCM_UNPACK_U8(pp, ver); 404 BCM_UNPACK_U32(pp, sig); 405 BCM_UNPACK_U16(pp, entry); 406 407 if (dir != 'C') { 408 LOG_ERROR(BSL_LS_BCM_COMMON, 409 (BSL_META("RPC: Old version %d entry %d non-request discarded\n"), 410 ver, entry)); 411 return; 412 } 413 414 LOG_ERROR(BSL_LS_BCM_COMMON, 415 (BSL_META("RPC: Old version %d entry %d request failed with BCM_E_UNAVAIL\n"), 416 ver, entry)); 417 418 /* construct a v1 RPC reply */ 419 r_pkt = atp_tx_data_alloc(BCM_RPC_V1_HLEN+4); 420 pp = r_pkt; 421 BCM_PACK_U32(pp, seq); 422 BCM_PACK_U8(pp, 'S'); 423 BCM_PACK_U8(pp, ver); 424 BCM_PACK_U32(pp, sig); 425 BCM_PACK_U16(pp, BCM_RPC_V1_HLEN+4); 426 BCM_PACK_U32(pp, BCM_E_UNAVAIL); 427 bcm_rpc_reply(r_cpu, r_pkt, pp-r_pkt); 428 } 429 430 #undef BCM_RX_DFLT_UNIT 431 432 433 static bcm_error_t 434 _bcm_rpc_sthread_db_update(uint32 oper, 435 _rpc_sthread_db_t *db) 436 { 437 uint32 tid; 438 bcm_error_t rv = BCM_E_NONE; 439 440 if (db == NULL) { 441 return BCM_E_PARAM; 442 } 443 444 switch (oper) { 445 case BCM_RPC_STHREAD_SET: 446 /* Update the Thread Database 447 */ 448 _rpc_s_db[db->thread_id].thread_id = db->thread_id; 449 _rpc_s_db[db->thread_id].status = db->status; 450 break; 451 case BCM_RPC_STHREAD_GET: 452 /* Get the Thread Status of the thread_id, 453 * passed as an argument. Currently Not used 454 */ 455 db->status = _rpc_s_db[db->thread_id].status; 456 break; 457 case BCM_RPC_STHREAD_NEXT: 458 /* If any free thread is available, return the next free thread 459 * else return the next thread. 460 */ 461 for (tid = 0; tid < _num_rpc_threads; tid++) { 462 if (_rpc_s_db[tid].status == BCM_RPC_SERVER_THREAD_FREE) { 463 db->thread_id = _rpc_s_db[tid].thread_id; 464 db->status = _rpc_s_db[tid].status; 465 _rpc_next_thread = _rpc_s_db[tid].thread_id; 466 break; 467 } 468 } 469 if (db->status != BCM_RPC_SERVER_THREAD_FREE) { 470 if ( ++_rpc_next_thread >= _num_rpc_threads ) { 471 _rpc_next_thread = 0; 472 } 473 db->thread_id = _rpc_s_db[_rpc_next_thread].thread_id; 474 db->status = _rpc_s_db[_rpc_next_thread].status; 475 } 476 477 break; 478 default: 479 /* Operation Not Supported 480 */ 481 rv = BCM_E_PARAM; 482 break; 483 } 484 return rv; 485 } 486 487 488 489 /* 490 * Packet handler. 491 * ATP callback function that handles receipt of both request and 492 * reply RPC packets. 493 */ 494 bcm_rx_t 495 bcm_rpc_pkt_handler(cpudb_key_t cpu, 496 int client_id, 497 bcm_pkt_t *pkt, 498 uint8 *buf, 499 int buf_len, 500 void *cookie) 501 { 502 uint32 seq; 503 uint32 thread_id; 504 uint8 dir, ver; 505 uint8 *bp; 506 int i; 507 _rpc_creq_t *req, *preq; 508 bcm_rpc_sreq_t *sreq; 509 _rpc_sthread_db_t db; 510 511 COMPILER_REFERENCE(client_id); 512 COMPILER_REFERENCE(buf_len); 513 COMPILER_REFERENCE(cookie); 514 515 /* decode header */ 516 bp = buf; 517 BCM_UNPACK_U32(bp, seq); 518 BCM_UNPACK_U8(bp, dir); 519 BCM_UNPACK_U8(bp, ver); 520 521 if ( dir == 'C' ) { 522 memset(&db, 0 ,sizeof(db)); 523 if (BCM_FAILURE( 524 _bcm_rpc_sthread_db_update(BCM_RPC_STHREAD_NEXT, &db))) 525 { 526 return BCM_RX_NOT_HANDLED; /* Should not enter here */ 527 } 528 thread_id = db.thread_id; 529 } else { 530 thread_id = 0; 531 } 532 533 if (ver != BCM_RPC_VERSION) { 534 RPC_COUNT(_rpc_count_s_wrongver); 535 if (ver == 1) { 536 _bcm_rpc_unavail_v1(cpu, buf, cookie); 537 return BCM_RX_HANDLED; 538 } 539 LOG_ERROR(BSL_LS_BCM_COMMON, 540 (BSL_META("RPC: Version %d packet received\n"), 541 ver)); 542 return BCM_RX_NOT_HANDLED; /* wrong version */ 543 } 544 545 switch (dir) { 546 case 'S': /* reply from server */ 547 /* find request in the list */ 548 RPC_CLOCK; 549 preq = NULL; 550 req = _rpc_creq; 551 while (req != NULL) { 552 if (req->seq == seq) { /* match: unlink request */ 553 if (preq == NULL) { 554 _rpc_creq = req->next; 555 } else { 556 preq->next = req->next; 557 } 558 break; 559 } 560 preq = req; 561 req = req->next; 562 } 563 RPC_CUNLOCK; 564 if (req != NULL) { 565 req->next = NULL; 566 req->rbuf = buf; 567 req->cookie = pkt; 568 sal_sem_give(req->sem); /* wake waiting thread */ 569 RPC_COUNT(_rpc_count_c_reply); 570 return BCM_RX_HANDLED_OWNED; 571 } 572 RPC_COUNT(_rpc_count_c_noreq); 573 return BCM_RX_HANDLED; /* no matching request */ 574 575 case 'C': /* request from client */ 576 /* add request to server thread's queue */ 577 RPC_COUNT(_rpc_count_s_request); 578 sreq = sal_alloc(sizeof(*sreq), "bcm_rpc_server_req"); 579 if (sreq == NULL) { 580 return BCM_RX_HANDLED; /* failure */ 581 } 582 sreq->next = NULL; 583 sreq->cpu = cpu; 584 sreq->buf = buf; 585 sreq->cookie = pkt; 586 bp += BCM_PACKLEN_U16; /* skip len */ 587 for (i = 0; i < BCM_RPC_LOOKUP_KEYLEN; i++) { 588 BCM_UNPACK_U32(bp, sreq->rpckey[i]); 589 } 590 RPC_SLOCK (thread_id); 591 if (_rpc_sreqs_tail [thread_id] == NULL) { 592 _rpc_sreqs [thread_id] = _rpc_sreqs_tail [thread_id] = sreq; 593 } else { 594 _rpc_sreqs_tail [thread_id] ->next = sreq; 595 _rpc_sreqs_tail [thread_id] = sreq; 596 } 597 RPC_SUNLOCK (thread_id); 598 /* Update the server DB , thread_id is already updated*/ 599 db.status = BCM_RPC_SERVER_THREAD_USED; 600 (void) _bcm_rpc_sthread_db_update(BCM_RPC_STHREAD_SET, &db); 601 RPC_WAKE (thread_id); /* wake up server */ 602 603 return BCM_RX_HANDLED_OWNED; 604 605 default: 606 RPC_COUNT(_rpc_count_s_wrongver); 607 LOG_ERROR(BSL_LS_BCM_COMMON, 608 (BSL_META("RPC: Version %d packet has unexpected direction (%d)\n"), 609 ver, dir)); 610 return BCM_RX_NOT_HANDLED; /* unknown direction */ 611 } 612 } 613 614 /* 615 * Binary search the server lookup table for a matching request key. 616 * Run the server rpc routine that is matched. 617 */ 618 void 619 bcm_rpc_run(bcm_rpc_sreq_t *sreq) 620 { 621 uint32 key0; 622 _bcm_server_routine_t rtn; 623 int lo, hi, new, i, match; 624 _bcm_server_lookup_t *sarr; 625 626 key0 = sreq->rpckey[0]; 627 sarr = _bcm_server_lookup; 628 lo = -1; 629 hi = BCM_RPC_LOOKUP_COUNT; 630 match = 0; 631 while (hi-lo > 1) { 632 new = (hi + lo) / 2; 633 if (sarr[new].key[0] > key0) { 634 hi = new; 635 } else if (sarr[new].key[0] < key0) { 636 lo = new; 637 } else { 638 /* key0 is equal, check the rest */ 639 match = 1; 640 for (i = 1; i < BCM_RPC_LOOKUP_KEYLEN; i++) { 641 if (sarr[new].key[i] > sreq->rpckey[i]) { 642 hi = new; 643 match = 0; 644 break; 645 } else if (sarr[new].key[i] < sreq->rpckey[i]) { 646 lo = new; 647 match = 0; 648 break; 649 } 650 } 651 if (match) { 652 lo = new; 653 break; 654 } 655 } 656 } 657 if (match) { 658 rtn = sarr[lo].routine; 659 } else { 660 RPC_COUNT(_rpc_count_s_nokey); 661 rtn = _bcm_server_unavail; 662 } 663 rtn(sreq->cpu, sreq->buf, sreq->cookie); 664 } 665 666 /* 667 * Server thread. 668 * Receives request packets, processes the each request. 669 * The process routines will send replies. 670 */ 671 void 672 bcm_rpc_thread(void *cookie) 673 { 674 bcm_rpc_sreq_t *sreq, *freq; 675 uint32 id = 0; 676 _rpc_sthread_db_t db; 677 678 679 COMPILER_REFERENCE(cookie); 680 if (cookie) { 681 id = *((uint32*) cookie); 682 } 683 684 _rpc_thread_exit [id] = 0; 685 for (;;) { 686 RPC_SLEEP (id); 687 688 /* grab a set of requests off the server queue */ 689 RPC_SLOCK (id); 690 sreq = _rpc_sreqs [id]; 691 _rpc_sreqs [id] = _rpc_sreqs_tail [id] = NULL; 692 RPC_SUNLOCK (id); 693 if (sreq == NULL) { /* nothing to do */ 694 if (_rpc_thread_exit [id]) { 695 _rpc_server_thread = SAL_THREAD_ERROR; 696 if (cookie) { 697 sal_free(cookie); 698 } 699 sal_thread_exit(0); 700 return; 701 } 702 continue; 703 } 704 705 /* run the requests */ 706 while (sreq != NULL) { 707 RPC_COUNT(_rpc_count_s_run); 708 bcm_rpc_run(sreq); 709 freq = sreq; 710 sreq = sreq->next; 711 sal_free(freq); 712 } 713 /* Update the Particular ThreadID is ready for use */ 714 /* Update the server DB with the corresponding thread_id and status. */ 715 db.thread_id = id; 716 db.status = BCM_RPC_SERVER_THREAD_FREE; 717 (void) _bcm_rpc_sthread_db_update(BCM_RPC_STHREAD_SET, &db); 718 } 719 } 720 721 int 722 bcm_rpc_start(void) 723 { 724 int rv, flags; 725 int thread_id; 726 int tmp; 727 char rpc_server_name [ 50 ]; 728 char *rpc_name; 729 uint32 *tid; 730 731 if (_rpc_server_thread != SAL_THREAD_ERROR) { 732 return BCM_E_BUSY; 733 } 734 735 736 737 if ( sizeof (rpc_server_name) < sizeof (_RPC_SERVER_NAME)) { 738 return BCM_E_INTERNAL; 739 } 740 /* Initialize the number of RPC Threads */ 741 _num_rpc_threads = soc_property_get(0, spn_RPC_SERVER_THREAD_COUNT, 742 RPC_SERVER_MIN_THREADS); 743 744 _rpc_server_locks = 745 sal_alloc(sizeof(*_rpc_server_locks)*_num_rpc_threads, 746 "bcm_rpc_server_locks"); 747 if (_rpc_server_locks == NULL) { 748 return BCM_E_MEMORY; 749 } 750 _rpc_thread_exit = 751 sal_alloc(sizeof(*_rpc_thread_exit)*_num_rpc_threads, 752 "bcm_rpc_exit_flag"); 753 if (_rpc_thread_exit == NULL) { 754 sal_free((void *)_rpc_server_locks); 755 return BCM_E_MEMORY; 756 } 757 _rpc_server_sems = 758 sal_alloc(sizeof(*_rpc_server_sems)*_num_rpc_threads, 759 "bcm_rpc_server_sems"); 760 if (_rpc_server_sems == NULL) { 761 sal_free((void *)_rpc_thread_exit); 762 sal_free((void *)_rpc_server_locks); 763 return BCM_E_MEMORY; 764 } 765 766 767 _rpc_sreqs = 768 sal_alloc(sizeof(*_rpc_sreqs)*_num_rpc_threads, 769 "bcm_rpc_sreqs"); 770 if (_rpc_sreqs == NULL) { 771 sal_free((void *)_rpc_server_sems); 772 sal_free((void *)_rpc_thread_exit); 773 sal_free((void *)_rpc_server_locks); 774 return BCM_E_MEMORY; 775 } 776 _rpc_sreqs_tail = 777 sal_alloc(sizeof(*_rpc_sreqs_tail)*_num_rpc_threads, 778 "bcm_rpc_sreqs_tail"); 779 if (_rpc_sreqs_tail == NULL) { 780 sal_free((void *)_rpc_sreqs); 781 sal_free((void *)_rpc_server_sems); 782 sal_free((void *)_rpc_thread_exit); 783 sal_free((void *)_rpc_server_locks); 784 return BCM_E_MEMORY; 785 } 786 787 _rpc_s_db = 788 sal_alloc(sizeof(*_rpc_s_db)*_num_rpc_threads, 789 "bcm_rpc_sthreads_db"); 790 if (_rpc_s_db == NULL) { 791 sal_free((void *)_rpc_sreqs_tail); 792 sal_free((void *)_rpc_sreqs); 793 sal_free((void *)_rpc_server_sems); 794 sal_free((void *)_rpc_thread_exit); 795 sal_free((void *)_rpc_server_locks); 796 return BCM_E_MEMORY; 797 } 798 799 _rpc_client_lock = sal_mutex_create("bcm_rpc_client"); 800 801 802 803 804 sal_strncpy ( rpc_server_name, _RPC_SERVER_NAME, sizeof (rpc_server_name)); 805 806 /* start all threads */ 807 for ( thread_id = 0; thread_id < _num_rpc_threads; ++thread_id ) { 808 _rpc_sreqs [ thread_id ] = _rpc_sreqs_tail [ thread_id ] = NULL; 809 810 rpc_name = rpc_server_name + _RPC_SERVER_NAME_EXT_PTR; 811 rpc_name [ 0 ] = '0' + (thread_id / 10); /* convert id to character */ 812 rpc_name [ 1 ] = '0' + (thread_id % 10); /* convert id to character */ 813 rpc_name [ 2 ] = '\0'; 814 815 _rpc_server_locks [thread_id] = sal_mutex_create(rpc_server_name); 816 _rpc_server_sems [thread_id] = sal_sem_create(rpc_server_name, sal_sem_BINARY, 0); 817 818 tid = sal_alloc(sizeof(*tid), rpc_server_name); 819 if (tid) { 820 *tid = thread_id; 821 _rpc_server_thread = sal_thread_create(rpc_server_name, 822 RPC_THREAD_STACK, 823 RPC_THREAD_PRIO, 824 bcm_rpc_thread, 825 tid); 826 } else { 827 _rpc_server_thread = SAL_THREAD_ERROR; 828 } 829 if (_rpc_server_thread == SAL_THREAD_ERROR) { 830 for ( tmp = 0; tmp < thread_id; ++tmp ) { 831 /* TBD... we need to stop running threads */ 832 sal_sem_destroy (_rpc_server_sems [tmp]); 833 sal_mutex_destroy (_rpc_server_locks [tmp]); 834 } 835 836 if (tid) { 837 sal_free ((void *)tid); 838 } 839 840 sal_free((void *)_rpc_s_db); 841 sal_free((void *)_rpc_sreqs_tail); 842 sal_free((void *)_rpc_sreqs); 843 sal_free((void *)_rpc_server_sems); 844 sal_free((void *)_rpc_thread_exit); 845 sal_free((void *)_rpc_server_locks); 846 847 sal_mutex_destroy(_rpc_client_lock); 848 _rpc_server_sems = NULL; 849 _rpc_thread_exit = NULL; 850 _rpc_server_locks = NULL; 851 _rpc_client_lock = NULL; 852 853 return BCM_E_RESOURCE; 854 } 855 /* Initialize the Server Data Base */ 856 _rpc_s_db[thread_id].thread_id = thread_id; 857 _rpc_s_db[thread_id].status = BCM_RPC_SERVER_THREAD_FREE; 858 } 859 860 /* can be configured to use nexthop or cpu2cpu */ 861 if (_rpc_nexthop) { 862 flags = ATP_F_REASSEM_BUF | ATP_F_NEXT_HOP; 863 } else { 864 flags = ATP_F_REASSEM_BUF; 865 } 866 rv = atp_register(RPC_CLIENT_ID, flags, 867 bcm_rpc_pkt_handler, NULL, -1, -1); 868 return rv; 869 } 870 871 int 872 bcm_rpc_stop(void) 873 { 874 _rpc_creq_t *req, *nreq; 875 int thread_id; 876 877 878 if (_rpc_server_thread == SAL_THREAD_ERROR) { 879 return BCM_E_NONE; 880 } 881 atp_unregister(RPC_CLIENT_ID); 882 883 /* destroy all threads */ 884 for ( thread_id = 0; thread_id < _num_rpc_threads; ++thread_id ) { 885 _rpc_thread_exit [thread_id] = 1; 886 RPC_WAKE (thread_id); 887 sal_thread_yield(); 888 while (_rpc_server_thread != SAL_THREAD_ERROR) { 889 RPC_WAKE (thread_id); 890 sal_usleep(10000); 891 } 892 /* make above while to work for the next thread */ 893 _rpc_server_thread = 0; 894 895 /* fail any queued client requests */ 896 RPC_CLOCK; 897 req = _rpc_creq; 898 _rpc_creq = NULL; 899 RPC_CUNLOCK; 900 while (req != NULL) { 901 nreq = req->next; 902 sal_sem_give(req->sem); /* wake waiting thread */ 903 req = nreq; 904 } 905 906 sal_sem_destroy(_rpc_server_sems [thread_id]); 907 sal_mutex_destroy(_rpc_server_locks [thread_id]); 908 } 909 _rpc_server_thread = SAL_THREAD_ERROR; 910 911 sal_free((void *)_rpc_s_db); 912 sal_free((void *)_rpc_sreqs_tail); 913 sal_free((void *)_rpc_sreqs); 914 sal_free((void *)_rpc_server_sems); 915 sal_free((void *)_rpc_thread_exit); 916 sal_free((void *)_rpc_server_locks); 917 918 _rpc_server_sems = NULL; 919 _rpc_thread_exit = NULL; 920 _rpc_server_locks = NULL; 921 922 sal_mutex_destroy(_rpc_client_lock); 923 _rpc_client_lock = NULL; 924 return BCM_E_NONE; 925 } 926 927 /* 928 * A bcm unit has been detached. Find any queued 929 * rpc requests for that unit and fail them. 930 * 931 * Note: A unit value of (-1) indicates to remove queued 932 * rpc request in all units. 933 */ 934 int 935 bcm_rpc_detach(int unit) 936 { 937 _rpc_creq_t *req, *nreq, *preq; 938 939 if (_rpc_client_lock == NULL) { 940 return BCM_E_UNAVAIL; 941 } 942 943 /* find request in the list */ 944 RPC_CLOCK; 945 preq = NULL; 946 req = _rpc_creq; 947 while (req != NULL) { 948 nreq = req->next; 949 if ((req->unit == unit) || (unit == -1)) { 950 if (preq == NULL) { 951 _rpc_creq = req->next; 952 } else { 953 preq->next = req->next; 954 } 955 req->next = NULL; 956 sal_sem_give(req->sem); /* wake waiting thread */ 957 RPC_COUNT(_rpc_count_c_detach); 958 } else { 959 preq = req; 960 } 961 req = nreq; 962 } 963 RPC_CUNLOCK; 964 return BCM_E_NONE; 965 } 966 967 /* 968 * Dispatchable attach/detach routines 969 */ 970 int 971 _bcm_client_attach(int unit, char *subtype) 972 { 973 cpudb_key_t *cpu; 974 int rv; 975 bcm_info_t info; 976 977 if (subtype == NULL) { 978 return BCM_E_CONFIG; 979 } 980 cpu = sal_alloc(sizeof(*cpu), "bcm_client_attach"); 981 if (cpu == NULL) { 982 return BCM_E_MEMORY; 983 } 984 sal_memset(cpu, 0, sizeof(*cpu)); 985 986 rv = cpudb_key_parse(subtype, cpu); 987 if (rv < 0) { 988 sal_free((void *)cpu); 989 return rv; 990 } 991 992 BCM_CONTROL(unit)->drv_control = (void *)cpu; 993 BCM_CONTROL(unit)->capability |= BCM_CAPA_REMOTE; 994 995 /* call bcm_client_info_get directly - do not dispatch */ 996 rv = bcm_client_info_get(unit, &info); 997 if (rv < 0) { 998 sal_free((void *)cpu); 999 BCM_CONTROL(unit)->drv_control = NULL; 1000 return rv; 1001 } 1002 BCM_CONTROL(unit)->chip_vendor = info.vendor; 1003 BCM_CONTROL(unit)->chip_device = info.device; 1004 BCM_CONTROL(unit)->chip_revision = info.revision; 1005 BCM_CONTROL(unit)->capability |= info.capability; 1006 1007 return BCM_E_NONE; 1008 } 1009 1010 int 1011 _bcm_client_detach(int unit) 1012 { 1013 /* free cpudb key */ 1014 if (BCM_CONTROL(unit)->drv_control != NULL) { 1015 sal_free(BCM_CONTROL(unit)->drv_control); 1016 } 1017 return BCM_E_NONE; 1018 } 1019 1020 /* match BCM control subtype strings for client types */ 1021 int 1022 _bcm_client_match(int unit, char *keystr_a, char *keystr_b) 1023 { 1024 cpudb_key_t a,b; 1025 1026 COMPILER_REFERENCE(unit); 1027 cpudb_key_parse(keystr_a, &a); 1028 cpudb_key_parse(keystr_b, &b); 1029 1030 return CPUDB_KEY_COMPARE(a, b); 1031 } 1032 1033 /* 1034 * Function: 1035 * bcm_rpc_cleanup 1036 * Purpose: 1037 * Clear pending RPC requests for specified destination CPU. 1038 * Parameters: 1039 * cpu - CPU key to clear pending requests for. 1040 * Returns: 1041 * BCM_E_NONE Success 1042 */ 1043 int 1044 bcm_rpc_cleanup(cpudb_key_t cpu) 1045 { 1046 int unit; 1047 cpudb_key_t *unit_cpu; 1048 1049 /* Clear pending RPC requests for all units residing in given CPU */ 1050 for (unit = 0; unit < BCM_CONTROL_MAX; unit++) { 1051 if (!BCM_UNIT_VALID(unit)) { 1052 continue; 1053 } 1054 /* Check for units matching cpu key */ 1055 if ((unit_cpu = (cpudb_key_t *)BCM_CONTROL(unit)->drv_control) 1056 == NULL) { 1057 continue; 1058 } 1059 if (CPUDB_KEY_COMPARE(*unit_cpu, cpu) == 0) { 1060 bcm_rpc_detach(unit); 1061 } 1062 } 1063 1064 return BCM_E_NONE; 1065 } 1066 1067 1068 #ifdef BROADCOM_DEBUG 1069 void 1070 bcm_rpc_dump(void) 1071 { 1072 _rpc_creq_t *creq; 1073 bcm_rpc_sreq_t *sreq; 1074 int i; 1075 char keybuf[CPUDB_KEY_STRING_LEN]; 1076 sal_usecs_t now; 1077 int thread_id; 1078 1079 /* Check if server thread is running*/ 1080 if (_rpc_server_thread == SAL_THREAD_ERROR) { 1081 LOG_CLI((BSL_META("RPC Server thread not running\n"))); 1082 return; 1083 } 1084 1085 LOG_CLI((BSL_META("RPC Client request %u reply %u fail %u timeout %u\n"), 1086 _rpc_count_c_request, 1087 _rpc_count_c_reply, 1088 _rpc_count_c_fail, 1089 _rpc_count_c_timeout)); 1090 LOG_CLI((BSL_META("RPC Client missing request %u detach remove %u seq %u\n"), 1091 _rpc_count_c_noreq, 1092 _rpc_count_c_detach, 1093 _rpc_seq)); 1094 LOG_CLI((BSL_META("RPC Server request %u reply %u run %u wrongver %u nokey %u " 1095 "repretry %u reperr %u memerr %u\n"), 1096 _rpc_count_s_request, 1097 _rpc_count_s_reply, 1098 _rpc_count_s_run, 1099 _rpc_count_s_wrongver, 1100 _rpc_count_s_nokey, 1101 _rpc_count_s_rretry, 1102 _rpc_count_s_rerr, 1103 _rpc_count_s_merr)); 1104 1105 now = sal_time_usecs(); 1106 LOG_CLI((BSL_META("RPC Client Requests: time now %u\n"), 1107 now)); 1108 i = 0; 1109 for (creq = _rpc_creq; creq != NULL; creq = creq->next) { 1110 i += 1; 1111 LOG_CLI((BSL_META("%d:\tunit %d seq %u cookie %p time %u (%u ago)\n"), 1112 i, creq->unit, creq->seq, creq->cookie, 1113 creq->time, now - creq->time)); 1114 } 1115 1116 LOG_CLI((BSL_META("RPC Server Requests:\n"))); 1117 i = 0; 1118 for ( thread_id = 0; thread_id < _num_rpc_threads; ++thread_id ) { 1119 LOG_CLI((BSL_META("RPC Server Requests for thread %d:\n"), thread_id)); 1120 i = 0; 1121 for (sreq = _rpc_sreqs [thread_id]; sreq != NULL; sreq = sreq->next) { 1122 i += 1; 1123 cpudb_key_format(sreq->cpu, keybuf, sizeof(keybuf)); 1124 LOG_CLI((BSL_META("%d:\tcpu %s buf %p cookie %p rpckey0 %x\n"), 1125 i, keybuf, sreq->buf, sreq->cookie, sreq->rpckey[0])); 1126 } 1127 } 1128 } 1129 #endif /* BROADCOM_DEBUG */ 1130 1131 #endif /* BCM_RPC_SUPPORT */