tdm_th2_ovsb.c (31423B)
1 /* 2 * 3 * This license is set out in https://raw.githubusercontent.com/Broadcom-Network-Switching-Software/OpenBCM/master/Legal/LICENSE file. 4 * 5 * Copyright 2007-2019 Broadcom Inc. All rights reserved. 6 * All Rights Reserved.$ 7 * 8 * TDM core oversub algorithms 9 */ 10 #ifdef _TDM_STANDALONE 11 #include <tdm_top.h> 12 #else 13 #include <soc/tdm/core/tdm_top.h> 14 #endif 15 16 /** 17 @name: tdm_th2_ovs_apply_constraints 18 @param: 19 20 Partition OVS PMs into 2 halfpipes 21 */ 22 int 23 tdm_th2_ovs_apply_constraints( tdm_mod_t *_tdm ) 24 { 25 int pms_per_pipe, pipe_id; 26 int pm_num, ln_num, i; 27 28 int speed_en_mtx[TH2_NUM_PHY_PM][6]; /* [pm_indx][speed_indx] 0 if <pm_indx> has <speed_indx> speed*/ 29 int speed_en_mtx_reduced[6]; 30 /* <speed_indx> 0: 10G; 1: 20G; 2: 25G; 3: 40G; 4: 50G; 5: 100G */ 31 int no_of_speeds_in_pipe; 32 int phy_base_port, phy_port; 33 int no_pms_hp[2]; 34 35 pipe_id = _tdm->_core_data.vars_pkg.cal_id; 36 37 pms_per_pipe = _tdm->_chip_data.soc_pkg.pm_num_phy_modules/TH2_NUM_QUAD; 38 39 /*TDM_PRINT2("tdm_th2_avs_part_halfpipe1() pipe_id=%d pms_per_pipe=%d \n",pipe_id, pms_per_pipe);*/ 40 41 for (i=0; i<6; i++) { 42 for (pm_num=0; pm_num<TH2_NUM_PHY_PM; pm_num++) { 43 speed_en_mtx[pm_num][i]=0; 44 } 45 speed_en_mtx_reduced[i]=0; 46 } 47 48 /* Build speed_en_mtx matrix*/ 49 for (pm_num=pipe_id*pms_per_pipe; pm_num<(pipe_id+1)*pms_per_pipe; pm_num++) { 50 _tdm->_chip_data.soc_pkg.soc_vars.th2.pm_ovs_halfpipe[pm_num] = -1; 51 phy_base_port = TH2_NUM_PM_LNS*pm_num+1; 52 for (ln_num=0; ln_num<_tdm->_chip_data.soc_pkg.pmap_num_lanes; ln_num++) { 53 phy_port = phy_base_port+ln_num; 54 if((_tdm->_chip_data.soc_pkg.speed[phy_port] != SPEED_0) && 55 ((_tdm->_chip_data.soc_pkg.state[phy_port-1] == PORT_STATE__OVERSUB) || 56 (_tdm->_chip_data.soc_pkg.state[phy_port-1] == PORT_STATE__OVERSUB_HG) ) ) { 57 /* Coding for pm_ovs_halfpipe: 58 -1 : not OVS; 59 2 : OVS but not assigned 60 0 : Half Pipe 0 61 1 : Half Pipe 1 62 */ 63 _tdm->_chip_data.soc_pkg.soc_vars.th2.pm_ovs_halfpipe[pm_num] = 2; /* OVS but not assigned */ 64 switch(_tdm->_chip_data.soc_pkg.speed[phy_port]) { 65 case SPEED_10G : 66 speed_en_mtx[pm_num][INDX_10G] = 1; break; 67 case SPEED_20G : 68 speed_en_mtx[pm_num][INDX_20G] = 1; break; 69 case SPEED_25G : 70 speed_en_mtx[pm_num][INDX_25G] = 1; break; 71 case SPEED_40G : 72 speed_en_mtx[pm_num][INDX_40G] = 1; break; 73 case SPEED_50G : 74 speed_en_mtx[pm_num][INDX_50G] = 1; break; 75 case SPEED_100G : 76 speed_en_mtx[pm_num][INDX_100G] = 1; break; 77 default : break; 78 } 79 } 80 } 81 for(i=0; i<6; i++) { 82 speed_en_mtx_reduced[i] = (speed_en_mtx[pm_num][i]==1) ? 1 : speed_en_mtx_reduced[i]; 83 } 84 } 85 86 no_of_speeds_in_pipe=0; 87 for (i=0; i<6; i++) { 88 if (speed_en_mtx_reduced[i]>0) { 89 no_of_speeds_in_pipe++; 90 } 91 } 92 93 no_pms_hp[0] = 0; no_pms_hp[1] = 0; 94 if (no_of_speeds_in_pipe > 5) { 95 /* Restriction 13:No port configurations with more than 4 port speed classes are supported. */ 96 TDM_ERROR2("tdm_th2_ovs_apply_constraints() PIPE %d No OVS port configurations with more than 4 port speed classes are supported; no_of_speeds_in_pipe=%d\n", pipe_id, no_of_speeds_in_pipe); 97 98 } 99 else if (no_of_speeds_in_pipe == 4) { 100 /*Restriction 14: The only supported port configurations with 4 port speed classes are: 101 10G/20G/40G/100G 102 10G/25G/50G/100G 103 */ 104 if ( (speed_en_mtx_reduced[INDX_20G] || speed_en_mtx_reduced[INDX_40G]) && (speed_en_mtx_reduced[INDX_25G] || speed_en_mtx_reduced[INDX_50G]) ) { 105 /* Group PMs with 25G/50G ports in HP0 and PMs with 20G/40G in HP1*/ 106 TDM_PRINT1("tdm_th2_ovs_apply_constraints() PIPE %d applying Restriction 14 \n",pipe_id); 107 for (pm_num=pipe_id*pms_per_pipe; pm_num<(pipe_id+1)*pms_per_pipe; pm_num++) { 108 if (speed_en_mtx[pm_num][INDX_25G] || speed_en_mtx[pm_num][INDX_50G]) { 109 if (no_pms_hp[0] < (pms_per_pipe/2)) { 110 _tdm->_chip_data.soc_pkg.soc_vars.th2.pm_ovs_halfpipe[pm_num] = 0; 111 no_pms_hp[0]++; 112 } 113 } else if (speed_en_mtx[pm_num][INDX_20G] || speed_en_mtx[pm_num][INDX_40G]) { 114 if (no_pms_hp[1] < (pms_per_pipe/2)) { 115 _tdm->_chip_data.soc_pkg.soc_vars.th2.pm_ovs_halfpipe[pm_num] = 1; 116 no_pms_hp[1]++; 117 } 118 } 119 } 120 } 121 } 122 else if (no_of_speeds_in_pipe > 1) { 123 /*Restriction 15: All port configurations with 1-3 port speed classes are supported, except 124 configurations that contain both 20G and 25G port speeds.*/ 125 if ( speed_en_mtx_reduced[INDX_20G] && speed_en_mtx_reduced[INDX_25G]) { 126 /* Group PMs with 25G/50G ports in HP0 and PMs with 20G/40G in HP1*/ 127 TDM_PRINT1("tdm_th2_ovs_apply_constraints() PIPE %d applying Restriction 15 \n",pipe_id); 128 for (pm_num=pipe_id*pms_per_pipe; pm_num<(pipe_id+1)*pms_per_pipe; pm_num++) { 129 if (speed_en_mtx[pm_num][INDX_25G]) { /*if (speed_en_mtx[pm_num][INDX_25G] || speed_en_mtx[pm_num][INDX_50G]) {*/ 130 if (no_pms_hp[0] < (pms_per_pipe/2)) { 131 _tdm->_chip_data.soc_pkg.soc_vars.th2.pm_ovs_halfpipe[pm_num] = 0; 132 no_pms_hp[0]++; 133 } 134 } else if (speed_en_mtx[pm_num][INDX_20G]) { /*} else if (speed_en_mtx[pm_num][INDX_20G] || speed_en_mtx[pm_num][INDX_40G]) {*/ 135 if (no_pms_hp[1] < (pms_per_pipe/2)) { 136 _tdm->_chip_data.soc_pkg.soc_vars.th2.pm_ovs_halfpipe[pm_num] = 1; 137 no_pms_hp[1]++; 138 } 139 } 140 } 141 } 142 } 143 else { 144 /* Do nothing; no constraints # of speeds is 0*/ 145 } 146 147 return PASS; 148 } 149 150 151 152 153 /** 154 @name: tdm_th2_ovs_part_halfpipe 155 @param: 156 157 Partition OVS PMs into 2 halfpipes 158 */ 159 int 160 tdm_th2_ovs_part_halfpipe( tdm_mod_t *_tdm ) 161 { 162 int pms_per_pipe, pipe_id; 163 int pm_speed[TH2_NUM_PHY_PM/TH2_NUM_QUAD]; 164 int pm_num_subports[TH2_NUM_PHY_PM/TH2_NUM_QUAD]; 165 int pm_num_sort[TH2_NUM_PHY_PM/TH2_NUM_QUAD]; 166 int pm_num, ln_num, pm_indx, i, j; 167 int phy_base_port, phy_port; 168 int hp0_speed, hp1_speed; 169 int no_pms_hp0, no_pms_hp1; 170 171 pipe_id = _tdm->_core_data.vars_pkg.cal_id; 172 173 pms_per_pipe = _tdm->_chip_data.soc_pkg.pm_num_phy_modules/TH2_NUM_QUAD; 174 175 /*TDM_PRINT2("tdm_th2_avs_part_halfpipe1() pipe_id=%d pms_per_pipe=%d \n",pipe_id, pms_per_pipe);*/ 176 177 pm_indx=0; 178 for (pm_num=pipe_id*pms_per_pipe; pm_num<(pipe_id+1)*pms_per_pipe; pm_num++) { 179 phy_base_port = TH2_NUM_PM_LNS*pm_num+1; 180 pm_speed[pm_indx] = 0; 181 pm_num_subports[pm_indx] = 0; 182 for (ln_num=0; ln_num<_tdm->_chip_data.soc_pkg.pmap_num_lanes; ln_num++) { 183 phy_port = phy_base_port+ln_num; 184 if((_tdm->_chip_data.soc_pkg.speed[phy_port] != SPEED_0) && 185 ((_tdm->_chip_data.soc_pkg.state[phy_port-1] == PORT_STATE__OVERSUB) || 186 (_tdm->_chip_data.soc_pkg.state[phy_port-1] == PORT_STATE__OVERSUB_HG) ) ) { 187 /*TDM_PRINT4("tdm_th2_avs_part_halfpipe2() pm_num=%d ln_num=%d port=%d pm_speed=%d\n",pm_num, ln_num, phy_port, _tdm->_chip_data.soc_pkg.speed[phy_port]);*/ 188 pm_speed[pm_indx] = pm_speed[pm_indx] + _tdm->_chip_data.soc_pkg.speed[phy_port]/1000; 189 pm_num_subports[pm_indx]++; 190 } 191 } 192 pm_num_sort[pm_indx] = pm_num; 193 /*TDM_PRINT4("tdm_th2_avs_part_halfpipe2() pm_indx=%d pm_num=%d pm_speed=%d pm_num_subports=%d\n",pm_indx, pm_num_sort[pm_indx], pm_speed[pm_indx], pm_num_subports[pm_indx]);*/ 194 pm_indx++; 195 } 196 197 /* Sort speeds of the PMs in descending order - bubble sort*/ 198 for (i=0; i<pms_per_pipe-1; i++) 199 { 200 for (j=pms_per_pipe-1; j>i; j--) 201 { 202 if ( (pm_speed[j] > pm_speed[j-1]) || ((pm_speed[j] == pm_speed[j-1]) && (pm_num_subports[j] > pm_num_subports[j-1]))) /* swap j with j-1*/ 203 { 204 int tmp; 205 tmp = pm_num_sort[j]; 206 pm_num_sort[j] = pm_num_sort[j-1]; 207 pm_num_sort[j-1] = tmp; 208 tmp = pm_speed[j]; 209 pm_speed[j] = pm_speed[j-1]; 210 pm_speed[j-1] = tmp; 211 tmp = pm_num_subports[j]; 212 pm_num_subports[j] = pm_num_subports[j-1]; 213 pm_num_subports[j-1] = tmp; 214 } 215 } 216 } 217 218 219 hp0_speed=0; 220 hp1_speed=0; 221 no_pms_hp0=0; 222 no_pms_hp1=0; 223 /* Compute HP 0 & 1 BW based on already allocated PMs in constraints*/ 224 for (i=0; i<pms_per_pipe; i++) { 225 pm_num = pm_num_sort[i]; 226 if (_tdm->_chip_data.soc_pkg.soc_vars.th2.pm_ovs_halfpipe[pm_num] == 0) { 227 hp0_speed += pm_speed[i]; 228 no_pms_hp0++; 229 } 230 if (_tdm->_chip_data.soc_pkg.soc_vars.th2.pm_ovs_halfpipe[pm_num] == 1) { 231 hp1_speed += pm_speed[i]; 232 no_pms_hp1++; 233 } 234 235 } 236 237 /* Do partition of the PMs into two half pipes */ 238 for (i=0; i<pms_per_pipe; i++) { 239 pm_num = pm_num_sort[i]; 240 if (_tdm->_chip_data.soc_pkg.soc_vars.th2.pm_ovs_halfpipe[pm_num] == 2) { /* is OVS unallocated PM */ 241 if ( ((no_pms_hp1 < (pms_per_pipe/2)) && (hp0_speed > hp1_speed)) || (no_pms_hp0 >= (pms_per_pipe/2)) ) 242 { 243 hp1_speed = hp1_speed + pm_speed[i]; 244 _tdm->_chip_data.soc_pkg.soc_vars.th2.pm_ovs_halfpipe[pm_num] = 1; 245 no_pms_hp1++; 246 } 247 else 248 { 249 hp0_speed = hp0_speed + pm_speed[i]; 250 _tdm->_chip_data.soc_pkg.soc_vars.th2.pm_ovs_halfpipe[pm_num] = 0; 251 no_pms_hp0++; 252 } 253 } 254 } 255 /*TDM_PRINT2("tdm_th2_avs_part_halfpipe3_1() hp0_speed=%d hp1_speed=%d\n", hp0_speed, hp1_speed);*/ 256 257 return PASS; 258 } 259 260 261 262 263 /** 264 @name: tdm_th2_ovs_fill_group 265 @param: 266 267 Partition OVS PMs into 2 halfpipes 268 */ 269 int 270 tdm_th2_ovs_fill_group( tdm_mod_t *_tdm) 271 { 272 int pms_per_pipe, pipe_id; 273 int pm_num_subports[TH2_NUM_PHY_PM/TH2_NUM_QUAD]; 274 int pm_num_sort[TH2_NUM_PHY_PM/TH2_NUM_QUAD]; 275 int pm_num, ln_num, pm_indx, i, j; 276 277 int pms_with_grp_speed; 278 int grp_speed; 279 int half_pipe_num; 280 int speed_max_num_ports_per_pm; 281 int min_num_ovs_groups; 282 tdm_calendar_t *cal; 283 int start_ovs_group; 284 285 286 pipe_id = _tdm->_core_data.vars_pkg.cal_id; 287 half_pipe_num = _tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num; 288 grp_speed = _tdm->_chip_data.soc_pkg.soc_vars.th2.grp_speed; 289 290 291 pms_per_pipe = _tdm->_chip_data.soc_pkg.pm_num_phy_modules/TH2_NUM_QUAD; 292 293 /*TDM_PRINT4("tdm_th2_ovs_fill_group1() pipe_id=%d pms_per_pipe=%d half_pipe_num=%d grp_speed=%d \n",pipe_id, pms_per_pipe, half_pipe_num, grp_speed);*/ 294 295 /* Count the number of subports per PM with grp_speed for PMs that belongs to this half pipe*/ 296 pm_indx=0; 297 pms_with_grp_speed = 0; 298 for (pm_num=pipe_id*pms_per_pipe; pm_num<(pipe_id+1)*pms_per_pipe; pm_num++) { 299 int phy_base_port = TH2_NUM_PM_LNS*pm_num+1; 300 pm_num_subports[pm_indx] = 0; 301 if (_tdm->_chip_data.soc_pkg.soc_vars.th2.pm_ovs_halfpipe[pm_num] == half_pipe_num){ 302 for (ln_num=0; ln_num<_tdm->_chip_data.soc_pkg.pmap_num_lanes; ln_num++) { 303 int phy_port = phy_base_port+ln_num; 304 if(_tdm->_chip_data.soc_pkg.speed[phy_port] == grp_speed) { 305 /*TDM_PRINT4("tdm_th2_ovs_fill_group2() pm_num=%d ln_num=%d port=%d half_pipe=%d\n",pm_num, ln_num, phy_port, half_pipe_num);*/ 306 pm_num_subports[pm_indx]++; 307 } 308 } 309 if (pm_num_subports[pm_indx] > 0) {pms_with_grp_speed++;} 310 } 311 pm_num_sort[pm_indx] = pm_num; 312 pm_indx++; 313 } 314 315 /* Compute the minimum number of oversub groups required for the group speed*/ 316 switch (grp_speed) { 317 case SPEED_10G: speed_max_num_ports_per_pm=4; break; 318 case SPEED_20G: speed_max_num_ports_per_pm=2; break; 319 case SPEED_40G: speed_max_num_ports_per_pm=2; break; 320 case SPEED_25G: speed_max_num_ports_per_pm=4; break; 321 case SPEED_50G: speed_max_num_ports_per_pm=2; break; 322 case SPEED_100G: speed_max_num_ports_per_pm=1; break; 323 default: 324 TDM_PRINT1("tdm_th2_ovs_fill_group3() Invalid group speed %0d\n",grp_speed); 325 return FAIL; 326 } 327 /* ceil [(speed_max_num_ports_per_pm*pms_with_grp_speed) / TH2_OS_VBS_GRP_LEN ]*/ 328 min_num_ovs_groups = ((speed_max_num_ports_per_pm*pms_with_grp_speed) + TH2_OS_VBS_GRP_LEN-1) / TH2_OS_VBS_GRP_LEN; 329 330 /* Execute only if ports with this speed exist*/ 331 if (min_num_ovs_groups > 0) { 332 int start_group; 333 int grp_index[TH2_OS_VBS_GRP_NUM]; 334 int phy_base_port; 335 int phy_port; 336 337 /* Sort PMs in descending order of their number of subports with grp_speed*/ 338 for (i=0; i<pms_per_pipe-1; i++) 339 { 340 for (j=pms_per_pipe-1; j>i; j--) 341 { 342 if ( pm_num_subports[j] > pm_num_subports[j-1]) /* swap j with j-1*/ 343 { 344 int tmp; 345 tmp = pm_num_sort[j]; 346 pm_num_sort[j] = pm_num_sort[j-1]; 347 pm_num_sort[j-1] = tmp; 348 tmp = pm_num_subports[j]; 349 pm_num_subports[j] = pm_num_subports[j-1]; 350 pm_num_subports[j-1] = tmp; 351 } 352 } 353 } 354 355 /* Find the first empty ovs group*/ 356 switch (_tdm->_core_data.vars_pkg.cal_id) { 357 case 0: cal=(&(_tdm->_chip_data.cal_0)); break; 358 case 1: cal=(&(_tdm->_chip_data.cal_1)); break; 359 case 2: cal=(&(_tdm->_chip_data.cal_2)); break; 360 case 3: cal=(&(_tdm->_chip_data.cal_3)); break; 361 case 4: cal=(&(_tdm->_chip_data.cal_4)); break; 362 case 5: cal=(&(_tdm->_chip_data.cal_5)); break; 363 case 6: cal=(&(_tdm->_chip_data.cal_6)); break; 364 case 7: cal=(&(_tdm->_chip_data.cal_7)); break; 365 default: 366 TDM_PRINT1("tdm_th2_ovs_fill_group()Invalid calendar ID - %0d\n",_tdm->_core_data.vars_pkg.cal_id); 367 return FAIL; 368 } 369 start_ovs_group = half_pipe_num*(TH2_OS_VBS_GRP_NUM/2); 370 start_group = start_ovs_group; 371 for (i=start_group; i< (start_group+(TH2_OS_VBS_GRP_NUM/2)); i++) { 372 if (cal->cal_grp[i][0] != TH2_NUM_EXT_PORTS) { 373 start_ovs_group++; 374 } else { break;} 375 } 376 377 378 /* Fill ovs groups with ports having grp_speed*/ 379 for (i=0; i< TH2_OS_VBS_GRP_NUM; i++) { grp_index[i] = 0;} 380 for (i=0; i<pms_with_grp_speed; i++) { 381 pm_num = pm_num_sort[i]; 382 phy_base_port = TH2_NUM_PM_LNS*pm_num+1; 383 if (_tdm->_chip_data.soc_pkg.soc_vars.th2.pm_ovs_halfpipe[pm_num] == half_pipe_num){ 384 for (ln_num=0; ln_num<_tdm->_chip_data.soc_pkg.pmap_num_lanes; ln_num++) { 385 phy_port = phy_base_port+ln_num; 386 if(_tdm->_chip_data.soc_pkg.speed[phy_port] == grp_speed) { 387 int t_ovs_grp_num; 388 t_ovs_grp_num = start_ovs_group + (i % min_num_ovs_groups); 389 cal->cal_grp[t_ovs_grp_num][grp_index[t_ovs_grp_num]] = phy_port; 390 grp_index[t_ovs_grp_num]++; 391 } 392 } 393 } 394 } 395 396 } 397 398 return PASS; 399 } 400 401 402 403 404 405 /** 406 @name: tdm_th2_ovs_pkt_shaper 407 @param: 408 409 Populates Pkt shaper calendar 410 */ 411 int 412 tdm_th2_ovs_pkt_shaper( tdm_mod_t *_tdm ) 413 { 414 int pms_per_pipe, pipe_id; 415 int pm_num, ln_num; 416 int phy_base_port, phy_port; 417 418 pipe_id = _tdm->_core_data.vars_pkg.cal_id; 419 420 pms_per_pipe = _tdm->_chip_data.soc_pkg.pm_num_phy_modules/TH2_NUM_QUAD; 421 422 423 for (pm_num=pipe_id*pms_per_pipe; pm_num<(pipe_id+1)*pms_per_pipe; pm_num++) { 424 phy_base_port = TH2_NUM_PM_LNS*pm_num+1; 425 for (ln_num=0; ln_num<_tdm->_chip_data.soc_pkg.pmap_num_lanes; ln_num++) { 426 phy_port = phy_base_port+ln_num; 427 if((_tdm->_chip_data.soc_pkg.speed[phy_port] != SPEED_0) && 428 ((_tdm->_chip_data.soc_pkg.state[phy_port-1] == PORT_STATE__OVERSUB) || 429 (_tdm->_chip_data.soc_pkg.state[phy_port-1] == PORT_STATE__OVERSUB_HG) ) ) { 430 /*TDM_PRINT4("tdm_th2_ovs_pkt_shaper2() pm_num=%d ln_num=%d port=%d pm_speed=%d\n",pm_num, ln_num, phy_port, _tdm->_chip_data.soc_pkg.speed[phy_port]);*/ 431 _tdm->_core_data.vars_pkg.port = phy_port; 432 tdm_th2_ovs_pkt_shaper_per_port(_tdm); 433 } 434 } 435 } 436 return PASS; 437 } 438 439 440 441 /** 442 @name: tdm_th2_ovs_pkt_shaper_per_port 443 @param: 444 445 Populates Pkt shaper calendar - per port 446 */ 447 int 448 tdm_th2_ovs_pkt_shaper_per_port( tdm_mod_t *_tdm ) 449 { 450 int pms_per_pipe; 451 int pm_num, i; 452 int phy_port; 453 int half_pipe_num; 454 455 int pkt_shpr_pm_indx; 456 int no_of_lanes; 457 int num_slots_for_port; 458 int total_num_slots; 459 int port_slot_tbl[SHAPING_GRP_LEN/8]; 460 int port_slot_tbl_shift[SHAPING_GRP_LEN/8]; 461 int slot_tbl_shift; 462 int subport_no; 463 int is_20G; 464 int *pkt_shed_cal; 465 int pkt_sched_repetitions; 466 int base_offset; 467 int max_pms_per_halfpipe; 468 469 phy_port = _tdm->_core_data.vars_pkg.port; 470 pm_num = (phy_port-1)/TH2_NUM_PM_LNS; 471 half_pipe_num = _tdm->_chip_data.soc_pkg.soc_vars.th2.pm_ovs_halfpipe[pm_num]; 472 473 pms_per_pipe = _tdm->_chip_data.soc_pkg.pm_num_phy_modules/TH2_NUM_QUAD; 474 max_pms_per_halfpipe = pms_per_pipe/2; 475 476 pkt_sched_repetitions = SHAPING_GRP_LEN/(pms_per_pipe/2)/TH2_NUM_PM_LNS; /* TH2: 5 = 160/8/4 */ 477 478 /*TDM_PRINT2("tdm_th2_ovs_pkt_shaper() pipe_id=%d pms_per_pipe=%d \n",pipe_id, pms_per_pipe);*/ 479 480 /* Determine the pm_indx in pkt shceduler calendar - 0 to 7*/ 481 pkt_shpr_pm_indx = tdm_th2_ovs_pkt_shaper_find_pm_indx(_tdm); 482 483 /* Find the number of lanes allocated to the port */ 484 is_20G=0; 485 switch (_tdm->_chip_data.soc_pkg.speed[phy_port]) { 486 case SPEED_10G: no_of_lanes=1; break; 487 case SPEED_20G: no_of_lanes=2; is_20G = 1; break; 488 case SPEED_40G: no_of_lanes=2; break; 489 case SPEED_25G: no_of_lanes=1; break; 490 case SPEED_50G: no_of_lanes=2; break; 491 case SPEED_100G: no_of_lanes=4; break; 492 default: 493 TDM_PRINT1("tdm_th2_ovs_fill_group3() Invalid group speed %0d\n",_tdm->_chip_data.soc_pkg.speed[phy_port]); 494 return FAIL; 495 } 496 497 /* Compute the number of slots in pkt calendar for this port*/ 498 num_slots_for_port = _tdm->_chip_data.soc_pkg.speed[phy_port]/5000; /* each slots is 5G */ 499 500 /* Compute the total number of slots consumed by the lanes*/ 501 total_num_slots = 5*no_of_lanes; 502 503 /* First, place num_slots_for_port slots in a table with total_num_slots equally spaced */ 504 for (i=0; i<SHAPING_GRP_LEN/max_pms_per_halfpipe; i++) { 505 port_slot_tbl[i] = TH2_NUM_EXT_PORTS; 506 port_slot_tbl_shift[i]= TH2_NUM_EXT_PORTS; 507 } 508 for (i=0; i<num_slots_for_port; i++) { 509 port_slot_tbl[(i*total_num_slots)/num_slots_for_port] = phy_port; 510 } 511 512 513 /* Get the right pkt scheduler calendar*/ 514 switch (_tdm->_core_data.vars_pkg.cal_id) { 515 case 0: pkt_shed_cal = _tdm->_chip_data.cal_0.cal_grp[SHAPING_GRP_IDX_0+half_pipe_num]; break; 516 case 1: pkt_shed_cal = _tdm->_chip_data.cal_1.cal_grp[SHAPING_GRP_IDX_0+half_pipe_num]; break; 517 case 2: pkt_shed_cal = _tdm->_chip_data.cal_2.cal_grp[SHAPING_GRP_IDX_0+half_pipe_num]; break; 518 case 3: pkt_shed_cal = _tdm->_chip_data.cal_3.cal_grp[SHAPING_GRP_IDX_0+half_pipe_num]; break; 519 default: 520 TDM_PRINT1("tdm_th2_ovs_pkt_shaper_per_port() Invalid calendar ID - %0d\n",_tdm->_core_data.vars_pkg.cal_id); 521 return FAIL; 522 } 523 524 /* In order to spread tokens as uniformly as possible shift slot_tbl based on subport number */ 525 subport_no = (phy_port-1)%TH2_NUM_PM_LNS; 526 slot_tbl_shift = 0; 527 if (no_of_lanes==1) { 528 switch (subport_no) { 529 case 0 : slot_tbl_shift=0; break; 530 case 1 : slot_tbl_shift=2; break; 531 case 2 : slot_tbl_shift=1; break; 532 case 3 : slot_tbl_shift=3; break; 533 default : slot_tbl_shift=0; break; 534 } 535 } 536 if (no_of_lanes==2) { 537 switch (subport_no) { 538 case 0 : slot_tbl_shift=0; break; 539 case 2 : slot_tbl_shift= (is_20G==1) ? 3 : 2; break; 540 default : slot_tbl_shift=0; break; 541 } 542 } 543 544 for (i=0; i<total_num_slots; i++) { 545 port_slot_tbl_shift[(i+slot_tbl_shift) % total_num_slots] = port_slot_tbl[i]; 546 } 547 548 549 /* Populate pkt scheduler calendar */ 550 base_offset = pkt_shpr_pm_indx % pkt_sched_repetitions; 551 for (i=0; i<total_num_slots; i++) { 552 int cal_pos; 553 int base_pos; 554 int lane, lane_pos; 555 556 lane = subport_no + (i%no_of_lanes); 557 /*base_pos = 32*(i/no_of_lanes);*/ 558 base_pos = 32*( (base_offset + (i/no_of_lanes)) % pkt_sched_repetitions); 559 if (no_of_lanes==4) { /* SINGLE port mode */ 560 lane_pos = max_pms_per_halfpipe*lane; 561 } else { /* DUAL, and QUAD port modes */ 562 switch (lane) { 563 case 0 : lane_pos = 0; break; 564 case 1 : lane_pos = 2*max_pms_per_halfpipe; break; 565 case 2 : lane_pos = max_pms_per_halfpipe; break; 566 case 3 : lane_pos = 3*max_pms_per_halfpipe; break; 567 default: 568 TDM_PRINT1("tdm_th2_ovs_pkt_shaper_per_port() phy_port lane for phy_port=%d\n",phy_port); 569 return FAIL; 570 } 571 } 572 573 cal_pos = base_pos + lane_pos + pkt_shpr_pm_indx; 574 pkt_shed_cal[cal_pos] = port_slot_tbl_shift[i]; 575 } 576 577 return PASS; 578 } 579 580 581 /** 582 @name: tdm_th2_ovs_pkt_shaper_find_pm_indx 583 @param: 584 585 Find an existing or available pm_indx in pkt_shaper 586 */ 587 int 588 tdm_th2_ovs_pkt_shaper_find_pm_indx( tdm_mod_t *_tdm ) 589 { 590 int pipe_id, pm_num, half_pipe_num; 591 int i, j, pm_num_t; 592 int phy_port; 593 int *pkt_shed_cal; 594 int pkt_shpr_pm_indx, pms_per_pipe; 595 int pm_indx_avail[8]; 596 int max_start_indx; 597 int distance; 598 int max_distance; 599 600 pipe_id = _tdm->_core_data.vars_pkg.cal_id; 601 phy_port = _tdm->_core_data.vars_pkg.port; 602 pm_num = tdm_th2_which_tsc(_tdm); 603 half_pipe_num = _tdm->_chip_data.soc_pkg.soc_vars.th2.pm_ovs_halfpipe[pm_num]; 604 pms_per_pipe = _tdm->_chip_data.soc_pkg.pm_num_phy_modules/TH2_NUM_QUAD; 605 606 607 /* Get the right pkt scheduler calendar*/ 608 switch (_tdm->_core_data.vars_pkg.cal_id) { 609 case 0: pkt_shed_cal = _tdm->_chip_data.cal_0.cal_grp[SHAPING_GRP_IDX_0+half_pipe_num]; break; 610 case 1: pkt_shed_cal = _tdm->_chip_data.cal_1.cal_grp[SHAPING_GRP_IDX_0+half_pipe_num]; break; 611 case 2: pkt_shed_cal = _tdm->_chip_data.cal_2.cal_grp[SHAPING_GRP_IDX_0+half_pipe_num]; break; 612 case 3: pkt_shed_cal = _tdm->_chip_data.cal_3.cal_grp[SHAPING_GRP_IDX_0+half_pipe_num]; break; 613 default: 614 TDM_PRINT1("tdm_th2_ovs_pkt_shaper_per_port() Invalid calendar ID - %0d\n",_tdm->_core_data.vars_pkg.cal_id); 615 return FAIL; 616 } 617 618 /* Two cases possible: 619 1. there are sister ports of this port already in the pkt scheduler; 620 which means find that PM and place phy_port in the corresponding lanes 621 2. there is no sister port of this port already in the pkt scheduler; 622 which means find an empty PM and place phy_port in the corresponding lanes*/ 623 624 pkt_shpr_pm_indx = -1; 625 for (i=0; i<SHAPING_GRP_LEN; i++) { 626 if (pkt_shed_cal[i] != _tdm->_chip_data.soc_pkg.num_ext_ports) { 627 _tdm->_core_data.vars_pkg.port = pkt_shed_cal[i]; 628 pm_num_t = tdm_th2_which_tsc(_tdm); 629 if (pm_num == pm_num_t) { /* Found that PM is already placed */ 630 pkt_shpr_pm_indx = (i% (pms_per_pipe/2)); /* i%8 */ 631 break; 632 } 633 } 634 } 635 636 /* Case 2: there is no sister port of this port already in the pkt scheduler; 637 which means find an empty PM and place phy_port in the corresponding lanes 638 Find an available PM */ 639 if (pkt_shpr_pm_indx == -1) { 640 for (i=0; i<(pms_per_pipe/2); i++){ 641 pm_indx_avail[i]=1; 642 for (j=0; j<(SHAPING_GRP_LEN/(pms_per_pipe/2)); j++){ 643 if (pkt_shed_cal[j*(pms_per_pipe/2)+i] != _tdm->_chip_data.soc_pkg.num_ext_ports) { 644 pm_indx_avail[i]=0; 645 break; 646 } 647 } 648 } 649 650 /* Find the biggest clump of 1's in pm_indx_avail array and choose the pm_indx in the middle of the clump*/ 651 max_start_indx = 0; 652 distance=0; 653 max_distance = 0; 654 for (i=0; i<(pms_per_pipe/2); i++){ 655 if (pm_indx_avail[i]==1) { 656 distance=0; 657 for (j=0; j<(pms_per_pipe/2); j++){ 658 if (pm_indx_avail[(i+j)%(pms_per_pipe/2)]==1) { 659 distance++; 660 } else { 661 break; 662 } 663 } 664 if (distance > max_distance) { 665 max_start_indx = i; 666 max_distance = distance; 667 } 668 } 669 } 670 /* If all available make it 0, else middle of the clump*/ 671 pkt_shpr_pm_indx = (max_distance==(pms_per_pipe/2)) ? 0 : ((max_start_indx + (max_distance/2)) % (pms_per_pipe/2)); 672 673 if (pm_indx_avail[pkt_shpr_pm_indx] == 0) { 674 pkt_shpr_pm_indx = -1; 675 TDM_ERROR3("tdm_th2_ovs_pkt_shaper_find_pm_indx() PIPE %d pm_num=%d phy_port=%d Unable to find an available PM \n",pipe_id, pm_num, phy_port); 676 } 677 } 678 679 if (pkt_shpr_pm_indx == -1) { 680 TDM_ERROR3("tdm_th2_ovs_pkt_shaper_find_pm_indx() PIPE %d pm_num=%d phy_port=%d Unable to find an available PM \n",pipe_id, pm_num, phy_port); 681 } 682 683 return pkt_shpr_pm_indx; 684 } 685 686 687 688 689 /** 690 @name: tdm_th2_ovs_map_pm_num_to_pblk 691 @param: 692 693 Maps PM num (block_id) to OVS pblk id 694 */ 695 int 696 tdm_th2_ovs_map_pm_num_to_pblk( tdm_mod_t *_tdm ) 697 { 698 int pms_per_pipe, pipe_id; 699 int pm_num; 700 int half_pipe_num; 701 int pblk_indx[2]; 702 703 pipe_id = _tdm->_core_data.vars_pkg.cal_id; 704 pms_per_pipe = _tdm->_chip_data.soc_pkg.pm_num_phy_modules/TH2_NUM_QUAD; 705 706 pblk_indx[0]=0; pblk_indx[1]=0; 707 for (pm_num=pipe_id*pms_per_pipe; pm_num<(pipe_id+1)*pms_per_pipe; pm_num++) { 708 _tdm->_chip_data.soc_pkg.soc_vars.th2.pm_num_to_pblk[pm_num] = -1; 709 half_pipe_num = _tdm->_chip_data.soc_pkg.soc_vars.th2.pm_ovs_halfpipe[pm_num]; 710 if ((half_pipe_num==0) || (half_pipe_num==1)){ 711 _tdm->_chip_data.soc_pkg.soc_vars.th2.pm_num_to_pblk[pm_num] = pblk_indx[half_pipe_num]; 712 pblk_indx[half_pipe_num]++; 713 } 714 } 715 return PASS; 716 } 717 718 719 720 721 /** 722 @name: tdm_th2_vbs_scheduler_ovs 723 @param: 724 725 Generate sortable weightable groups for oversub round robin arbitration 726 */ 727 int 728 tdm_th2_vbs_scheduler_ovs( tdm_mod_t *_tdm ) 729 { 730 tdm_calendar_t *cal; 731 732 733 switch (_tdm->_core_data.vars_pkg.cal_id) { 734 case 0: cal=(&(_tdm->_chip_data.cal_0)); break; 735 case 1: cal=(&(_tdm->_chip_data.cal_1)); break; 736 case 2: cal=(&(_tdm->_chip_data.cal_2)); break; 737 case 3: cal=(&(_tdm->_chip_data.cal_3)); break; 738 case 4: cal=(&(_tdm->_chip_data.cal_4)); break; 739 case 5: cal=(&(_tdm->_chip_data.cal_5)); break; 740 case 6: cal=(&(_tdm->_chip_data.cal_6)); break; 741 case 7: cal=(&(_tdm->_chip_data.cal_7)); break; 742 default: 743 TDM_PRINT1("Invalid calendar ID - %0d\n",_tdm->_core_data.vars_pkg.cal_id); 744 return (TDM_EXEC_CORE_SIZE+1); 745 } 746 TDM_BIG_BAR 747 TDM_PRINT9("(1G - %0d) (10G - %0d) (20G - %0d) (25G - %0d) (40G - %0d) (50G - %0d) (100G - %0d) (120G - %0d) (Number of Oversub Types - %0d)\n", _tdm->_core_data.vars_pkg.m_tdm_core_vbs_scheduler.z8, _tdm->_core_data.vars_pkg.m_tdm_core_vbs_scheduler.z1, _tdm->_core_data.vars_pkg.m_tdm_core_vbs_scheduler.z2, _tdm->_core_data.vars_pkg.m_tdm_core_vbs_scheduler.z6, _tdm->_core_data.vars_pkg.m_tdm_core_vbs_scheduler.z3, _tdm->_core_data.vars_pkg.m_tdm_core_vbs_scheduler.z5, _tdm->_core_data.vars_pkg.m_tdm_core_vbs_scheduler.z4, _tdm->_core_data.vars_pkg.m_tdm_core_vbs_scheduler.z7, (_tdm->_core_data.vars_pkg.os_1 + _tdm->_core_data.vars_pkg.os_10 + _tdm->_core_data.vars_pkg.os_20 + _tdm->_core_data.vars_pkg.os_25 + _tdm->_core_data.vars_pkg.os_40 + _tdm->_core_data.vars_pkg.os_50 + _tdm->_core_data.vars_pkg.os_100 + _tdm->_core_data.vars_pkg.os_120)); 748 if ((_tdm->_core_data.vars_pkg.os_1 + _tdm->_core_data.vars_pkg.os_10 + _tdm->_core_data.vars_pkg.os_20 + _tdm->_core_data.vars_pkg.os_25 + _tdm->_core_data.vars_pkg.os_40 + _tdm->_core_data.vars_pkg.os_50 + _tdm->_core_data.vars_pkg.os_100 + _tdm->_core_data.vars_pkg.os_120) > cal->grp_num) { 749 TDM_ERROR0("Oversub speed type limit exceeded\n"); 750 return FAIL; 751 } 752 if (((_tdm->_core_data.vars_pkg.os_1 + _tdm->_core_data.vars_pkg.os_10 + _tdm->_core_data.vars_pkg.os_20 + _tdm->_core_data.vars_pkg.os_25 + _tdm->_core_data.vars_pkg.os_40 + _tdm->_core_data.vars_pkg.os_50 + _tdm->_core_data.vars_pkg.os_100)==cal->grp_num && (_tdm->_core_data.vars_pkg.m_tdm_core_vbs_scheduler.z6>cal->grp_len || _tdm->_core_data.vars_pkg.m_tdm_core_vbs_scheduler.z1>cal->grp_len || _tdm->_core_data.vars_pkg.m_tdm_core_vbs_scheduler.z2>cal->grp_len)) || ((_tdm->_core_data.vars_pkg.os_1 + _tdm->_core_data.vars_pkg.os_10 + _tdm->_core_data.vars_pkg.os_20 + _tdm->_core_data.vars_pkg.os_25 + _tdm->_core_data.vars_pkg.os_40 + _tdm->_core_data.vars_pkg.os_50 + _tdm->_core_data.vars_pkg.os_100)>=(cal->grp_num-1) && (_tdm->_core_data.vars_pkg.m_tdm_core_vbs_scheduler.z6>32 || _tdm->_core_data.vars_pkg.m_tdm_core_vbs_scheduler.z1>32 || _tdm->_core_data.vars_pkg.m_tdm_core_vbs_scheduler.z2>32)) || ((_tdm->_core_data.vars_pkg.os_1 + _tdm->_core_data.vars_pkg.os_10 + _tdm->_core_data.vars_pkg.os_20 + _tdm->_core_data.vars_pkg.os_25 + _tdm->_core_data.vars_pkg.os_40 + _tdm->_core_data.vars_pkg.os_50 + _tdm->_core_data.vars_pkg.os_100)>=(cal->grp_num-2) && (_tdm->_core_data.vars_pkg.m_tdm_core_vbs_scheduler.z6>48 || _tdm->_core_data.vars_pkg.m_tdm_core_vbs_scheduler.z1>48)) || ((_tdm->_core_data.vars_pkg.os_1 + _tdm->_core_data.vars_pkg.os_10 + _tdm->_core_data.vars_pkg.os_20 + _tdm->_core_data.vars_pkg.os_25 + _tdm->_core_data.vars_pkg.os_40 + _tdm->_core_data.vars_pkg.os_50 + _tdm->_core_data.vars_pkg.os_100)>=(cal->grp_num-3) && (_tdm->_core_data.vars_pkg.m_tdm_core_vbs_scheduler.z6>TDM_AUX_SIZE || _tdm->_core_data.vars_pkg.m_tdm_core_vbs_scheduler.z1>TDM_AUX_SIZE))) { 753 TDM_ERROR0("Oversub bucket overflow\n"); 754 return FAIL; 755 } 756 757 if(tdm_th2_ovs_apply_constraints(_tdm)) 758 TDM_PRINT1("OVS Apply constraints PIPE %d DONE\n", _tdm->_core_data.vars_pkg.cal_id); 759 760 if(tdm_th2_ovs_part_halfpipe(_tdm)) 761 TDM_PRINT1("OVS PM Half Pipe partition for PIPE %d DONE\n", _tdm->_core_data.vars_pkg.cal_id); 762 763 764 _tdm->_chip_data.soc_pkg.soc_vars.th2.grp_speed = SPEED_10G; 765 _tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num = 0; 766 if(tdm_th2_ovs_fill_group(_tdm)) { 767 TDM_PRINT2("OVS partition for PIPE %d HALF_PIPE=%d DONE\n", _tdm->_core_data.vars_pkg.cal_id,_tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num); 768 } 769 _tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num = 1; 770 if(tdm_th2_ovs_fill_group(_tdm)) { 771 TDM_PRINT2("OVS partition for PIPE %d HALF_PIPE=%d DONE\n", _tdm->_core_data.vars_pkg.cal_id,_tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num); 772 } 773 774 _tdm->_chip_data.soc_pkg.soc_vars.th2.grp_speed = SPEED_20G; 775 _tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num = 0; 776 if(tdm_th2_ovs_fill_group(_tdm)) { 777 TDM_PRINT2("OVS partition for PIPE %d HALF_PIPE=%d DONE\n", _tdm->_core_data.vars_pkg.cal_id,_tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num); 778 } 779 _tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num = 1; 780 if(tdm_th2_ovs_fill_group(_tdm)) { 781 TDM_PRINT2("OVS partition for PIPE %d HALF_PIPE=%d DONE\n", _tdm->_core_data.vars_pkg.cal_id,_tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num); 782 } 783 784 _tdm->_chip_data.soc_pkg.soc_vars.th2.grp_speed = SPEED_25G; 785 _tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num = 0; 786 if(tdm_th2_ovs_fill_group(_tdm)) { 787 TDM_PRINT2("OVS partition for PIPE %d HALF_PIPE=%d DONE\n", _tdm->_core_data.vars_pkg.cal_id,_tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num); 788 } 789 _tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num = 1; 790 if(tdm_th2_ovs_fill_group(_tdm)) { 791 TDM_PRINT2("OVS partition for PIPE %d HALF_PIPE=%d DONE\n", _tdm->_core_data.vars_pkg.cal_id,_tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num); 792 } 793 794 _tdm->_chip_data.soc_pkg.soc_vars.th2.grp_speed = SPEED_40G; 795 _tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num = 0; 796 if(tdm_th2_ovs_fill_group(_tdm)) { 797 TDM_PRINT2("OVS partition for PIPE %d HALF_PIPE=%d DONE\n", _tdm->_core_data.vars_pkg.cal_id,_tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num); 798 } 799 _tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num = 1; 800 if(tdm_th2_ovs_fill_group(_tdm)) { 801 TDM_PRINT2("OVS partition for PIPE %d HALF_PIPE=%d DONE\n", _tdm->_core_data.vars_pkg.cal_id,_tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num); 802 } 803 804 _tdm->_chip_data.soc_pkg.soc_vars.th2.grp_speed = SPEED_50G; 805 _tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num = 0; 806 if(tdm_th2_ovs_fill_group(_tdm)) { 807 TDM_PRINT2("OVS partition for PIPE %d HALF_PIPE=%d DONE\n", _tdm->_core_data.vars_pkg.cal_id,_tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num); 808 } 809 _tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num = 1; 810 if(tdm_th2_ovs_fill_group(_tdm)) { 811 TDM_PRINT2("OVS partition for PIPE %d HALF_PIPE=%d DONE\n", _tdm->_core_data.vars_pkg.cal_id,_tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num); 812 } 813 814 _tdm->_chip_data.soc_pkg.soc_vars.th2.grp_speed = SPEED_100G; 815 _tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num = 0; 816 if(tdm_th2_ovs_fill_group(_tdm)) { 817 TDM_PRINT2("OVS partition for PIPE %d HALF_PIPE=%d DONE\n", _tdm->_core_data.vars_pkg.cal_id,_tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num); 818 } 819 _tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num = 1; 820 if(tdm_th2_ovs_fill_group(_tdm)) { 821 TDM_PRINT2("OVS partition for PIPE %d HALF_PIPE=%d DONE\n", _tdm->_core_data.vars_pkg.cal_id,_tdm->_chip_data.soc_pkg.soc_vars.th2.half_pipe_num); 822 } 823 824 if(tdm_th2_ovs_pkt_shaper(_tdm)) { 825 TDM_PRINT1("OVS partition for PIPE %d PKT_SHAPER CALENDAR DONE\n", _tdm->_core_data.vars_pkg.cal_id); 826 } 827 828 if(tdm_th2_ovs_map_pm_num_to_pblk(_tdm)) { 829 TDM_PRINT1("OVS partition for PIPE %d PM_NUM to PBLK mapping DONE\n", _tdm->_core_data.vars_pkg.cal_id); 830 } 831 832 return PASS; 833 834 835 836 837 }