tokenizer.c (14210B)
1 /* 2 * 3 * This license is set out in https://raw.githubusercontent.com/Broadcom-Network-Switching-Software/OpenBCM/master/Legal/LICENSE file. 4 * 5 * Copyright 2007-2019 Broadcom Inc. All rights reserved. 6 * 7 * File: tokenizer.c 8 * Purpose: API mode tokenizer 9 */ 10 11 /* 12 // 13 // dot format tokenizer state machine 14 // 15 // character classes 16 // W = whitespace 17 // Q = quote characters 18 // Q' = starting quote character 19 // E = escape character '\' 20 // S = separator ;=?.{} 21 // T = token character !(W|Q|E) 22 // C = comment character 23 // 0 = end of string 24 25 // + = allocate token 26 // - = delimit token 27 28 digraph tokenizer { 29 start -> search 30 search -> search [label="W"] 31 search -> quote [label="Q+"] 32 search -> error [label="E"] 33 search -> token [label="T+"] 34 search -> sep [label="S+"] 35 search -> comment [label="C+"] 36 search -> end [label="0-"] 37 quote -> quote [label="!Q'"] 38 quote -> delim [label="Q'"] 39 quote -> error [label="0-"] 40 delim -> search [label="W-"] 41 delim -> sep [label="S-+"] 42 delim -> error [label="T|Q|E"] 43 delim -> comment [label="C+"] 44 delim -> end [label="0-"] 45 token -> token [label="T"] 46 token -> search [label="W-"] 47 token -> sep [label="S-+"] 48 token -> comment [label="C+"] 49 token -> error [label="Q|E"] 50 token -> end [label="0-"] 51 sep -> search [label="W-"] 52 sep -> token [label="T-+"] 53 sep -> quote [label="Q-+"] 54 sep -> sep [label="S-+"] 55 sep -> comment [label="C+"] 56 sep -> error [label="E"] 57 sep -> end [label="0-"] 58 comment -> comment [label="!(0|N)"] 59 comment -> search [label="N"] 60 comment -> end [label="0"] 61 } 62 */ 63 64 #include "sal/core/alloc.h" 65 #include "sal/core/libc.h" 66 #include "tokenizer.h" 67 68 typedef enum { 69 SEARCH, 70 TOKEN, 71 QUOTE, 72 DELIM, 73 SEP, 74 COMMENT, 75 TKERR /* Something in vxWorks defined an ERROR macro */ 76 } tkstate_t; 77 78 typedef enum { 79 C, /* comment */ 80 E, /* escape */ 81 N, /* newline */ 82 Q, /* quote */ 83 S, /* separator */ 84 T, /* token */ 85 W /* whitespace */ 86 } tkchar_t; 87 88 /* 89 Tokenize input string. 90 91 An unquoted token is a contiguous sequence of non-whitespace 92 characters, except single quote (') douple quote (") and backslash 93 (\). 94 95 A quoted token is delimited by single or double quotes. A quote 96 character may be escaped with a backslash. 97 98 It is an error for a backslash character to appear outside of a 99 quoted token, or for a quoted token to be immediately followed by 100 another token (quoted or not) with no intervening whitespace. 101 102 Returns 1 if the tokenization completed without error, or zero if 103 there was an error. Tokenization stops when an error is detected, so 104 if there is an error, the last token is the erroneous one. 105 106 input: input string to tokenize 107 tokens: token buffer 108 write: true to write tokens out, false to only count 109 110 */ 111 112 /* return true if the character is the beginning of an identifier */ 113 STATIC int 114 _is_ident(int c) 115 { 116 return ((c == '_') || 117 (c >= 'A' && c <= 'Z') || 118 (c >= 'a' && c <= 'z')); 119 } 120 121 /* 122 * Function: 123 * _api_mode_tokenizer(const char *input, 124 * api_mode_tokens_t *tokens, int write) 125 * 126 * Purpose: 127 * Tokenize a string. 128 * Parameters: 129 * input - (IN) Input string 130 * tokens - (IN/OUT) Token structure 131 * write - (IN) Create tokens if true, Count tokens if false 132 * Returns: 133 * 0 - no errors 134 * -1 - parse error 135 * Notes: 136 * Tokenizer kernel. 137 */ 138 139 STATIC int 140 _api_mode_tokenizer(const char *input, api_mode_tokens_t *tokens, int write) 141 { 142 tkstate_t state; 143 tkchar_t cclass; 144 int c; 145 int start_quote; 146 int escape; 147 int delim; 148 int token; 149 int line; 150 int new_line; 151 int first_line; 152 int first_column; 153 int last_column; 154 int ident; 155 const char *base; 156 char *p; 157 api_mode_token_type_t token_type; 158 159 p = (write) ? tokens->buf : NULL; 160 tokens->len = 0; 161 state = SEARCH; 162 start_quote = 0; 163 escape = 0; 164 delim = 0; 165 token = 0; 166 line = 0; 167 new_line = 0; 168 first_line = 0; 169 first_column = 0; 170 ident = 0; 171 base = input; 172 token_type = API_MODE_TOKEN_TYPE_ERROR; 173 tokens->error = API_MODE_TOKEN_ERROR_INIT; 174 175 for (;(c=*input) && state != TKERR; input++) { 176 177 if (new_line) { 178 line++; 179 base = input; 180 new_line = 0; 181 } 182 183 /* classify character */ 184 if (c <= ' ') { 185 cclass = W; 186 if (c == '\n') { 187 new_line = 1; 188 } 189 } else if (c == '"' || c == '\'') { 190 cclass = Q; 191 } else if ( c == ';' 192 || c == '=' 193 || c == '?' 194 || c == '.' 195 || c == '!' 196 || c == '@' 197 || c == '$' 198 || c == '%' 199 || c == '^' 200 || c == '&' 201 || c == '*' 202 || c == '+' 203 || c == '-' 204 || c == '~' 205 || c == '`' 206 || c == '|' 207 || c == ':' 208 || c == ',' 209 || c == '/' 210 || c == '{' 211 || c == '}' 212 || c == '(' 213 || c == ')' 214 || c == '<' 215 || c == '>' 216 || c == '[' 217 || c == ']') { 218 cclass = S; 219 } else if (c == '\\') { 220 cclass = E; 221 } else if (c == '#') { 222 cclass = C; 223 } else { 224 cclass = T; 225 } 226 227 switch (state) { 228 case SEARCH: 229 if (cclass == T) { 230 token = 1; 231 token_type = API_MODE_TOKEN_TYPE_VALUE; 232 state = TOKEN; 233 ident = _is_ident(c); 234 } else if (cclass == S) { 235 delim = 1; 236 token = 1; 237 token_type = API_MODE_TOKEN_TYPE_SEPARATOR; 238 state = SEP; 239 } else if (cclass == Q) { 240 start_quote = c; 241 token = 1; 242 token_type = API_MODE_TOKEN_TYPE_QUOTE; 243 state = QUOTE; 244 } else if (cclass == C) { 245 delim = 1; 246 token = 1; 247 token_type = API_MODE_TOKEN_TYPE_COMMENT; 248 state = COMMENT; 249 } else if (cclass == E) { 250 tokens->error = API_MODE_TOKEN_ERROR_ESCAPE; 251 state = TKERR; 252 } 253 break; 254 case TOKEN: 255 if (cclass == W) { 256 delim = 1; 257 state = SEARCH; 258 } else if (cclass == S) { 259 delim = 1; 260 token = 1; 261 token_type = API_MODE_TOKEN_TYPE_SEPARATOR; 262 state = SEP; 263 } else if (cclass == C) { 264 delim = 1; 265 token = 1; 266 token_type = API_MODE_TOKEN_TYPE_COMMENT; 267 state = COMMENT; 268 } else if (cclass == Q) { 269 tokens->error = API_MODE_TOKEN_ERROR_QUOTE; 270 state = TKERR; 271 } else if (cclass == E) { 272 tokens->error = API_MODE_TOKEN_ERROR_ESCAPE; 273 state = TKERR; 274 } 275 break; 276 case QUOTE: 277 if (escape) { 278 escape = 0; 279 } else if (c == start_quote) { 280 state = DELIM; 281 } else if (cclass == E) { 282 escape = 1; 283 } 284 break; 285 case DELIM: 286 if (cclass == W) { 287 delim = 1; 288 state = SEARCH; 289 } else if (cclass == S) { 290 delim = 1; 291 token = 1; 292 token_type = API_MODE_TOKEN_TYPE_SEPARATOR; 293 state = SEP; 294 } else if (cclass == C) { 295 delim = 1; 296 token = 1; 297 token_type = API_MODE_TOKEN_TYPE_COMMENT; 298 state = COMMENT; 299 } else if (cclass == Q) { 300 tokens->error = API_MODE_TOKEN_ERROR_QUOTE; 301 state = TKERR; 302 } else if (cclass == E) { 303 tokens->error = API_MODE_TOKEN_ERROR_ESCAPE; 304 state = TKERR; 305 } else { 306 tokens->error = API_MODE_TOKEN_ERROR_CHAR; 307 state = TKERR; 308 } 309 break; 310 case SEP: 311 if (cclass == W) { 312 delim = 1; 313 state = SEARCH; 314 } else if (cclass == S) { 315 delim = 1; 316 token = 1; 317 token_type = API_MODE_TOKEN_TYPE_SEPARATOR; 318 state = SEP; 319 } else if (cclass == Q) { 320 delim = 1; 321 token = 1; 322 token_type = API_MODE_TOKEN_TYPE_QUOTE; 323 state = QUOTE; 324 start_quote = c; 325 } else if (cclass == T) { 326 delim = 1; 327 token = 1; 328 token_type = API_MODE_TOKEN_TYPE_VALUE; 329 state = TOKEN; 330 ident = _is_ident(c); 331 } else if (cclass == C) { 332 delim = 1; 333 token = 1; 334 token_type = API_MODE_TOKEN_TYPE_COMMENT; 335 state = COMMENT; 336 } else { 337 /* should never get here unless there's an error in 338 character classification */ 339 tokens->error = API_MODE_TOKEN_ERROR_UNEXPECTED; 340 state = TKERR; 341 } 342 break; 343 case COMMENT: 344 /* just throw away characters until newline or done */ 345 if (new_line) { 346 delim = 1; 347 state = SEARCH; 348 } 349 break; 350 default: 351 /* should never get here unless a state got added but not 352 handled */ 353 tokens->error = API_MODE_TOKEN_ERROR_STATE; 354 state = TKERR; 355 /* fall through */ 356 case TKERR: 357 /* should never get here unless the loop fails to exit on 358 the error state. */ 359 break; 360 } 361 362 /* delimit */ 363 if (p && delim) { 364 *p++ = 0; 365 delim = 0; 366 } 367 368 /* new token */ 369 if (token) { 370 if (write) { 371 last_column = input - base; 372 tokens->token[tokens->len].str = p; 373 tokens->token[tokens->len].first_line = first_line; 374 tokens->token[tokens->len].first_column = first_column; 375 tokens->token[tokens->len].last_line = line; 376 tokens->token[tokens->len].last_column = last_column; 377 tokens->token[tokens->len].token_type = token_type; 378 tokens->token[tokens->len].ident = 379 (token_type == API_MODE_TOKEN_TYPE_VALUE) && ident; 380 first_line = line; 381 first_column = last_column+1; 382 383 } 384 tokens->len++; 385 token = 0; 386 } 387 388 if (p && state != SEARCH) { 389 *p++ = c; 390 } 391 } 392 393 /* delimit final string */ 394 if (p && ((p > tokens->buf && p[-1] != 0) || (p == tokens->buf))) { 395 *p = 0; 396 } 397 398 if (state != TKERR && state != QUOTE) { 399 tokens->error = API_MODE_TOKEN_ERROR_NONE; 400 } 401 402 return tokens->error; 403 } 404 405 /* 406 * Function: 407 * api_mode_tokenizer(const char *input, 408 * api_mode_tokens_t *tokens) 409 * 410 * Purpose: 411 * Tokenize a string. 412 * Parameters: 413 * input - (IN) Input string 414 * tokens - (IN/OUT) Token structure 415 * Returns: 416 * 0 - no errors 417 * -1 - memory or parse error 418 * Notes: 419 * Top level tokenizer. Call api_mode_tokenizer_free() when 420 * done with the token structure. 421 */ 422 423 int 424 api_mode_tokenizer(const char *input, api_mode_tokens_t *tokens) 425 { 426 int rv,buf_len, alloc_len; 427 428 /* first pass to count characters */ 429 if ((rv=_api_mode_tokenizer(input, tokens, 0)) == 0) { 430 buf_len = alloc_len = (sal_strlen(input)+1)*2; 431 alloc_len += (tokens->len * sizeof(tokens->token[0])); 432 433 tokens->buf = sal_alloc(alloc_len, "tokenizer"); 434 if (tokens->buf) { 435 char *token_buf = tokens->buf + buf_len; 436 tokens->token = (api_mode_token_t *)token_buf; 437 /* second pass to actually tokenize */ 438 rv = _api_mode_tokenizer(input, tokens, 1); 439 } else { 440 rv = -1; 441 } 442 } else { 443 tokens->buf = NULL; 444 } 445 446 return rv; 447 } 448 449 /* 450 * Function: 451 * api_mode_tokenizer_free(api_mode_tokens_t *tokens) 452 * 453 * Purpose: 454 * Free allocated token data 455 * Parameters: 456 * tokens - (IN) Token structure 457 * Returns: 458 * 0 - no errors 459 * -1 - memory or parse error 460 * Notes: 461 * Top level tokenizer. Call api_mode_tokenizer_free() when 462 * done with the token structure. 463 */ 464 465 int 466 api_mode_tokenizer_free(api_mode_tokens_t *tokens) 467 { 468 if (tokens->buf) { 469 sal_free(tokens->buf); 470 } 471 472 return 0; 473 } 474 475 STATIC const char *tokenizer_error[] = { 476 "no error", 477 "never progressed beyond initial state", 478 "escape character unexpected", 479 "quote character unexpected", 480 "non-delimiter character unexpected", 481 "character classification error", 482 "unknown tokenizer state", 483 "unknown error" 484 }; 485 486 /* 487 * Function: 488 * api_mode_tokenizer_error_str(api_mode_token_error_t error) 489 * 490 * Purpose: 491 * Return an error string corresponding to a token error number. 492 * Parameters: 493 * error - (IN) token error number 494 * Returns: 495 * string 496 */ 497 498 const char * 499 api_mode_tokenizer_error_str(api_mode_token_error_t error) 500 { 501 if (error < 0 || error > API_MODE_TOKEN_ERROR_LAST) { 502 error = API_MODE_TOKEN_ERROR_LAST; 503 } 504 return tokenizer_error[error]; 505 }