From fd7807e92d8ba5740615362b453bab2d9ecbc8a4 Mon Sep 17 00:00:00 2001 From: Sean Barrett Date: Sun, 11 Jul 2021 16:37:34 -0700 Subject: [PATCH] stb_c_lexer: allow including stb_c_lexer.h without defining overrides (all tokens are always defined; token values have changed) --- stb_c_lexer.h | 93 ++++++++++------------------------ tests/test_c_lexer.c | 49 ++++++++++++++++++ tests/test_cpp_compilation.cpp | 49 ------------------ 3 files changed, 77 insertions(+), 114 deletions(-) diff --git a/stb_c_lexer.h b/stb_c_lexer.h index f8da55d..8d4305f 100644 --- a/stb_c_lexer.h +++ b/stb_c_lexer.h @@ -42,6 +42,7 @@ // // See end of file for license information. +#ifdef STB_C_LEXER_IMPLEMENTATION #ifndef STB_C_LEXER_DEFINITIONS // to change the default parsing rules, copy the following lines // into your C/C++ file *before* including this, and then replace @@ -54,7 +55,6 @@ #error "Can only use stb_c_lexer in contexts where the preprocessor symbols 'Y' and 'N' are not defined" #endif - #define STB_C_LEX_C_DECIMAL_INTS Y // "0|[1-9][0-9]*" CLEX_intlit #define STB_C_LEX_C_HEX_INTS Y // "0x[0-9a-fA-F]+" CLEX_intlit #define STB_C_LEX_C_OCTAL_INTS Y // "[0-7]+" CLEX_intlit @@ -102,7 +102,7 @@ #define STB_C_LEXER_DEFINITIONS // This line prevents the header file from replacing your definitions // --END-- - +#endif #endif #ifndef INCLUDE_STB_C_LEXER_H @@ -172,78 +172,41 @@ extern void stb_c_lexer_get_location(const stb_lexer *lexer, const char *where, } #endif -// Hacky definitions so we can easily #if on them -#define Y(x) 1 -#define N(x) 0 - -// Config variable that influence which lexer tokens get declared need to go here -#if STB_C_LEX_C_DECIMAL_INTS(x) || STB_C_LEX_C_HEX_INTS(x) || STB_C_LEX_DEFINE_ALL_TOKEN_NAMES(x) -#define STB__clex_define_int -#endif - -#if (STB_C_LEX_C_ARITHEQ(x) && STB_C_LEX_C_SHIFTS(x)) || STB_C_LEX_DEFINE_ALL_TOKEN_NAMES(x) -#define STB__clex_define_shifts -#endif - -// Now pick a definition of Y/N that's conducive to -// defining the enum of token names. -#if STB_C_LEX_DEFINE_ALL_TOKEN_NAMES(x) || defined(STB_C_LEXER_SELF_TEST) - #undef N - #define N(a) Y(a) -#else - #undef N - #define N(a) -#endif - -#undef Y -#define Y(a) a, - enum { CLEX_eof = 256, CLEX_parse_error, - -#ifdef STB__clex_define_int - CLEX_intlit, -#endif - - STB_C_LEX_C_DECIMAL_FLOATS( CLEX_floatlit ) - STB_C_LEX_C_IDENTIFIERS( CLEX_id ) - STB_C_LEX_C_DQ_STRINGS( CLEX_dqstring ) - STB_C_LEX_C_SQ_STRINGS( CLEX_sqstring ) - STB_C_LEX_C_CHARS( CLEX_charlit ) - STB_C_LEX_C_COMPARISONS( CLEX_eq ) - STB_C_LEX_C_COMPARISONS( CLEX_noteq ) - STB_C_LEX_C_COMPARISONS( CLEX_lesseq ) - STB_C_LEX_C_COMPARISONS( CLEX_greatereq ) - STB_C_LEX_C_LOGICAL( CLEX_andand ) - STB_C_LEX_C_LOGICAL( CLEX_oror ) - STB_C_LEX_C_SHIFTS( CLEX_shl ) - STB_C_LEX_C_SHIFTS( CLEX_shr ) - STB_C_LEX_C_INCREMENTS( CLEX_plusplus ) - STB_C_LEX_C_INCREMENTS( CLEX_minusminus ) - STB_C_LEX_C_ARITHEQ( CLEX_pluseq ) - STB_C_LEX_C_ARITHEQ( CLEX_minuseq ) - STB_C_LEX_C_ARITHEQ( CLEX_muleq ) - STB_C_LEX_C_ARITHEQ( CLEX_diveq ) - STB_C_LEX_C_ARITHEQ( CLEX_modeq ) - STB_C_LEX_C_BITWISEEQ( CLEX_andeq ) - STB_C_LEX_C_BITWISEEQ( CLEX_oreq ) - STB_C_LEX_C_BITWISEEQ( CLEX_xoreq ) - STB_C_LEX_C_ARROW( CLEX_arrow ) - STB_C_LEX_EQUAL_ARROW( CLEX_eqarrow ) - -#ifdef STB__clex_define_shifts + CLEX_intlit , + CLEX_floatlit , + CLEX_id , + CLEX_dqstring , + CLEX_sqstring , + CLEX_charlit , + CLEX_eq , + CLEX_noteq , + CLEX_lesseq , + CLEX_greatereq , + CLEX_andand , + CLEX_oror , + CLEX_shl , + CLEX_shr , + CLEX_plusplus , + CLEX_minusminus , + CLEX_pluseq , + CLEX_minuseq , + CLEX_muleq , + CLEX_diveq , + CLEX_modeq , + CLEX_andeq , + CLEX_oreq , + CLEX_xoreq , + CLEX_arrow , + CLEX_eqarrow , CLEX_shleq, CLEX_shreq, -#endif CLEX_first_unused_token }; - -#undef Y -#undef N - #endif // INCLUDE_STB_C_LEXER_H #ifdef STB_C_LEXER_IMPLEMENTATION diff --git a/tests/test_c_lexer.c b/tests/test_c_lexer.c index 579ed5d..cc11234 100644 --- a/tests/test_c_lexer.c +++ b/tests/test_c_lexer.c @@ -1 +1,50 @@ #include "stb_c_lexer.h" + +#define STB_C_LEX_C_DECIMAL_INTS Y // "0|[1-9][0-9]*" CLEX_intlit +#define STB_C_LEX_C_HEX_INTS Y // "0x[0-9a-fA-F]+" CLEX_intlit +#define STB_C_LEX_C_OCTAL_INTS Y // "[0-7]+" CLEX_intlit +#define STB_C_LEX_C_DECIMAL_FLOATS Y // "[0-9]*(.[0-9]*([eE][-+]?[0-9]+)?) CLEX_floatlit +#define STB_C_LEX_C99_HEX_FLOATS N // "0x{hex}+(.{hex}*)?[pP][-+]?{hex}+ CLEX_floatlit +#define STB_C_LEX_C_IDENTIFIERS Y // "[_a-zA-Z][_a-zA-Z0-9]*" CLEX_id +#define STB_C_LEX_C_DQ_STRINGS Y // double-quote-delimited strings with escapes CLEX_dqstring +#define STB_C_LEX_C_SQ_STRINGS N // single-quote-delimited strings with escapes CLEX_ssstring +#define STB_C_LEX_C_CHARS Y // single-quote-delimited character with escape CLEX_charlits +#define STB_C_LEX_C_COMMENTS Y // "/* comment */" +#define STB_C_LEX_CPP_COMMENTS Y // "// comment to end of line\n" +#define STB_C_LEX_C_COMPARISONS Y // "==" CLEX_eq "!=" CLEX_noteq "<=" CLEX_lesseq ">=" CLEX_greatereq +#define STB_C_LEX_C_LOGICAL Y // "&&" CLEX_andand "||" CLEX_oror +#define STB_C_LEX_C_SHIFTS Y // "<<" CLEX_shl ">>" CLEX_shr +#define STB_C_LEX_C_INCREMENTS Y // "++" CLEX_plusplus "--" CLEX_minusminus +#define STB_C_LEX_C_ARROW Y // "->" CLEX_arrow +#define STB_C_LEX_EQUAL_ARROW N // "=>" CLEX_eqarrow +#define STB_C_LEX_C_BITWISEEQ Y // "&=" CLEX_andeq "|=" CLEX_oreq "^=" CLEX_xoreq +#define STB_C_LEX_C_ARITHEQ Y // "+=" CLEX_pluseq "-=" CLEX_minuseq + // "*=" CLEX_muleq "/=" CLEX_diveq "%=" CLEX_modeq + // if both STB_C_LEX_SHIFTS & STB_C_LEX_ARITHEQ: + // "<<=" CLEX_shleq ">>=" CLEX_shreq + +#define STB_C_LEX_PARSE_SUFFIXES N // letters after numbers are parsed as part of those numbers, and must be in suffix list below +#define STB_C_LEX_DECIMAL_SUFFIXES "" // decimal integer suffixes e.g. "uUlL" -- these are returned as-is in string storage +#define STB_C_LEX_HEX_SUFFIXES "" // e.g. "uUlL" +#define STB_C_LEX_OCTAL_SUFFIXES "" // e.g. "uUlL" +#define STB_C_LEX_FLOAT_SUFFIXES "" // + +#define STB_C_LEX_0_IS_EOF Y // if Y, ends parsing at '\0'; if N, returns '\0' as token +#define STB_C_LEX_INTEGERS_AS_DOUBLES N // parses integers as doubles so they can be larger than 'int', but only if STB_C_LEX_STDLIB==N +#define STB_C_LEX_MULTILINE_DSTRINGS N // allow newlines in double-quoted strings +#define STB_C_LEX_MULTILINE_SSTRINGS N // allow newlines in single-quoted strings +#define STB_C_LEX_USE_STDLIB Y // use strtod,strtol for parsing #s; otherwise inaccurate hack +#define STB_C_LEX_DOLLAR_IDENTIFIER Y // allow $ as an identifier character +#define STB_C_LEX_FLOAT_NO_DECIMAL Y // allow floats that have no decimal point if they have an exponent + +#define STB_C_LEX_DEFINE_ALL_TOKEN_NAMES N // if Y, all CLEX_ token names are defined, even if never returned + // leaving it as N should help you catch config bugs + +#define STB_C_LEX_DISCARD_PREPROCESSOR Y // discard C-preprocessor directives (e.g. after prepocess + // still have #line, #pragma, etc) + +//#define STB_C_LEX_ISWHITE(str) ... // return length in bytes of whitespace characters if first char is whitespace + +#define STB_C_LEXER_DEFINITIONS // This line prevents the header file from replacing your definitions + +#include "stb_c_lexer.h" diff --git a/tests/test_cpp_compilation.cpp b/tests/test_cpp_compilation.cpp index 66f8f85..fd8c5b6 100644 --- a/tests/test_cpp_compilation.cpp +++ b/tests/test_cpp_compilation.cpp @@ -183,53 +183,4 @@ void dummy3(void) stb_textedit_paste(0,0,0,0); } -#if 0 -#define STB_C_LEX_C_DECIMAL_INTS Y // "0|[1-9][0-9]*" CLEX_intlit -#define STB_C_LEX_C_HEX_INTS Y // "0x[0-9a-fA-F]+" CLEX_intlit -#define STB_C_LEX_C_OCTAL_INTS Y // "[0-7]+" CLEX_intlit -#define STB_C_LEX_C_DECIMAL_FLOATS Y // "[0-9]*(.[0-9]*([eE][-+]?[0-9]+)?) CLEX_floatlit -#define STB_C_LEX_C99_HEX_FLOATS N // "0x{hex}+(.{hex}*)?[pP][-+]?{hex}+ CLEX_floatlit -#define STB_C_LEX_C_IDENTIFIERS Y // "[_a-zA-Z][_a-zA-Z0-9]*" CLEX_id -#define STB_C_LEX_C_DQ_STRINGS Y // double-quote-delimited strings with escapes CLEX_dqstring -#define STB_C_LEX_C_SQ_STRINGS N // single-quote-delimited strings with escapes CLEX_ssstring -#define STB_C_LEX_C_CHARS Y // single-quote-delimited character with escape CLEX_charlits -#define STB_C_LEX_C_COMMENTS Y // "/* comment */" -#define STB_C_LEX_CPP_COMMENTS Y // "// comment to end of line\n" -#define STB_C_LEX_C_COMPARISONS Y // "==" CLEX_eq "!=" CLEX_noteq "<=" CLEX_lesseq ">=" CLEX_greatereq -#define STB_C_LEX_C_LOGICAL Y // "&&" CLEX_andand "||" CLEX_oror -#define STB_C_LEX_C_SHIFTS Y // "<<" CLEX_shl ">>" CLEX_shr -#define STB_C_LEX_C_INCREMENTS Y // "++" CLEX_plusplus "--" CLEX_minusminus -#define STB_C_LEX_C_ARROW Y // "->" CLEX_arrow -#define STB_C_LEX_EQUAL_ARROW N // "=>" CLEX_eqarrow -#define STB_C_LEX_C_BITWISEEQ Y // "&=" CLEX_andeq "|=" CLEX_oreq "^=" CLEX_xoreq -#define STB_C_LEX_C_ARITHEQ Y // "+=" CLEX_pluseq "-=" CLEX_minuseq - // "*=" CLEX_muleq "/=" CLEX_diveq "%=" CLEX_modeq - // if both STB_C_LEX_SHIFTS & STB_C_LEX_ARITHEQ: - // "<<=" CLEX_shleq ">>=" CLEX_shreq - -#define STB_C_LEX_PARSE_SUFFIXES N // letters after numbers are parsed as part of those numbers, and must be in suffix list below -#define STB_C_LEX_DECIMAL_SUFFIXES "" // decimal integer suffixes e.g. "uUlL" -- these are returned as-is in string storage -#define STB_C_LEX_HEX_SUFFIXES "" // e.g. "uUlL" -#define STB_C_LEX_OCTAL_SUFFIXES "" // e.g. "uUlL" -#define STB_C_LEX_FLOAT_SUFFIXES "" // - -#define STB_C_LEX_0_IS_EOF Y // if Y, ends parsing at '\0'; if N, returns '\0' as token -#define STB_C_LEX_INTEGERS_AS_DOUBLES N // parses integers as doubles so they can be larger than 'int', but only if STB_C_LEX_STDLIB==N -#define STB_C_LEX_MULTILINE_DSTRINGS N // allow newlines in double-quoted strings -#define STB_C_LEX_MULTILINE_SSTRINGS N // allow newlines in single-quoted strings -#define STB_C_LEX_USE_STDLIB Y // use strtod,strtol for parsing #s; otherwise inaccurate hack -#define STB_C_LEX_DOLLAR_IDENTIFIER Y // allow $ as an identifier character -#define STB_C_LEX_FLOAT_NO_DECIMAL Y // allow floats that have no decimal point if they have an exponent - -#define STB_C_LEX_DEFINE_ALL_TOKEN_NAMES N // if Y, all CLEX_ token names are defined, even if never returned - // leaving it as N should help you catch config bugs - -#define STB_C_LEX_DISCARD_PREPROCESSOR Y // discard C-preprocessor directives (e.g. after prepocess - // still have #line, #pragma, etc) - -//#define STB_C_LEX_ISWHITE(str) ... // return length in bytes of whitespace characters if first char is whitespace - -#define STB_C_LEXER_DEFINITIONS // This line prevents the header file from replacing your definitions -#endif - #include "stb_c_lexer.h" -- GitLab