From 3e5c8dd80de98ce4b6e5bdb65cd0c4af8a9eceb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Villemot?= Date: Mon, 24 Sep 2018 17:57:55 +0200 Subject: [PATCH] Macro-processor: implement comprehensions Due to a limitation of the current implementation, this breaks syntaxes like [ (i,j) ] (but not [ (2,j) ]; the problem only occurs when an array is constructed by specifying as first element a tuple whose first element is a variable name). Solving this problem requires an overhaul of the macro-processor, with construction of ASTs at parsing time, and evaluation later on (instead of doing on-the-fly evaluation). Ref #5 --- src/macro/MacroBison.yy | 38 +++++++++++++++++- src/macro/MacroDriver.cc | 73 ++++++++++++++++++++++++++++++++++ src/macro/MacroDriver.hh | 36 ++++++++++++++++- src/macro/MacroFlex.ll | 86 ++++++++++++++++++++++++++++++++++++++-- 4 files changed, 226 insertions(+), 7 deletions(-) diff --git a/src/macro/MacroBison.yy b/src/macro/MacroBison.yy index 9d48a907..fbf415b3 100644 --- a/src/macro/MacroBison.yy +++ b/src/macro/MacroBison.yy @@ -68,6 +68,7 @@ class MacroDriver; %token COMMA DEFINE LINE FOR IN IF ECHO_DIR ERROR IFDEF IFNDEF POWER %token LPAREN RPAREN LBRACKET RBRACKET EQUAL EOL LENGTH ECHOMACROVARS SAVE +%token SEMICOLON ATSIGN %token INTEGER %token NAME STRING @@ -88,7 +89,7 @@ class MacroDriver; %type > comma_name %type expr -%type > comma_expr tuple_comma_expr +%type > comma_expr tuple_comma_expr comprehension_clause %% %start statement_list_or_nothing; @@ -217,6 +218,24 @@ expr : INTEGER { TYPERR_CATCH($$ = $1->set_intersection($3), @$); } | expr POWER expr { TYPERR_CATCH($$ = $1->power($3), @$); } + | LBRACKET NAME IN expr SEMICOLON + { + driver.init_comprehension(vector{$2}, $4); + driver.iter_comprehension(); + } + comprehension_clause RBRACKET + { + $$ = make_shared($7); + } + | LBRACKET LPAREN comma_name RPAREN IN expr SEMICOLON + { + driver.init_comprehension($3, $6); + driver.iter_comprehension(); + } + comprehension_clause RBRACKET + { + $$ = make_shared($9); + } ; comma_expr : %empty @@ -237,6 +256,23 @@ tuple_comma_expr : %empty { $1.push_back($3); $$ = $1; } ; +/* The lexer will repeat the comprehension clause as many times as there are + elements in the set to be filtered. It also adds a dummy at-sign (@) at the + end of every repetition (for making parsing of repetitions unambiguous). */ +comprehension_clause : expr ATSIGN + { + $$ = vector{}; + driver.possibly_add_comprehension_element($$, $1); + driver.iter_comprehension(); + } + | comprehension_clause expr ATSIGN + { + $$ = $1; + driver.possibly_add_comprehension_element($$, $2); + driver.iter_comprehension(); + } + ; + %% void diff --git a/src/macro/MacroDriver.cc b/src/macro/MacroDriver.cc index 0798ba5d..017cd95b 100644 --- a/src/macro/MacroDriver.cc +++ b/src/macro/MacroDriver.cc @@ -260,6 +260,79 @@ MacroDriver::iter_loop() noexcept(false) } } +void +MacroDriver::init_comprehension(const vector &names, MacroValuePtr value) +{ + auto mv = dynamic_pointer_cast(value); + if (!mv) + throw MacroValue::TypeError("In a comprehension, the expression after the 'in' keyword must be an array"); + comprehension_stack.emplace(names, move(mv), 0); +} + +int +MacroDriver::get_comprehension_iter_nb() const +{ + assert(!comprehension_stack.empty()); + + auto &mv = get<1>(comprehension_stack.top()); + return mv->values.size(); +} + + +void +MacroDriver::iter_comprehension() +{ + assert(!comprehension_stack.empty()); + + int &i = get<2>(comprehension_stack.top()); + auto &mv = get<1>(comprehension_stack.top()); + vector &names = get<0>(comprehension_stack.top()); + + assert(i <= static_cast(mv->values.size())); + + if (i == static_cast(mv->values.size())) + comprehension_stack.pop(); + else + { + if (names.size() == 1) + env[names.at(0)] = mv->values[i++]; + else + { + auto tmv = dynamic_pointer_cast(mv->values[i++]); + if (!tmv) + throw MacroValue::TypeError("The expression after the 'in' keyword must be an array expression of tuples"); + if (tmv->values.size() != names.size()) + { + cerr << "Error in comprehension loop: tuple in array contains " << tmv->length() + << " elements while you are assigning to " << names.size() << " variables." + << endl; + exit(EXIT_FAILURE); + } + + for (auto &name: names) + { + auto idx = &name - &names[0]; + env[name] = tmv->values.at(idx); + } + } + } +} + +void +MacroDriver::possibly_add_comprehension_element(vector &v, MacroValuePtr test_expr) const +{ + auto ival = dynamic_pointer_cast(test_expr); + if (!ival) + throw MacroValue::TypeError("In a comprehension, the expression after the 'if' must evaluate to an integer"); + if (ival->value) + { + assert(!comprehension_stack.empty()); + const int &i = get<2>(comprehension_stack.top()); + auto &mv = get<1>(comprehension_stack.top()); + v.push_back(mv->values.at(i-1)); + } +} + void MacroDriver::begin_if(const MacroValuePtr &value) noexcept(false) { diff --git a/src/macro/MacroDriver.hh b/src/macro/MacroDriver.hh index cc6dc909..db6f42a4 100644 --- a/src/macro/MacroDriver.hh +++ b/src/macro/MacroDriver.hh @@ -60,12 +60,27 @@ private: const bool is_for_context; const string for_body; const Macro::parser::location_type for_body_loc; + const bool is_comprehension_context; + const string comprehension_clause; + const Macro::parser::location_type comprehension_clause_loc; + const int comprehension_start_condition; + const int comprehension_iter_nb; ScanContext(istream *input_arg, struct yy_buffer_state *buffer_arg, Macro::parser::location_type &yylloc_arg, bool is_for_context_arg, string for_body_arg, - Macro::parser::location_type &for_body_loc_arg) : + Macro::parser::location_type &for_body_loc_arg, + bool is_comprehension_context_arg, + string comprehension_clause_arg, + Macro::parser::location_type &comprehension_clause_loc_arg, + int comprehension_start_condition_arg, + int comprehension_iter_nb_arg) : input(input_arg), buffer(buffer_arg), yylloc(yylloc_arg), is_for_context(is_for_context_arg), - for_body(move(for_body_arg)), for_body_loc(for_body_loc_arg) + for_body(move(for_body_arg)), for_body_loc(for_body_loc_arg), + is_comprehension_context{is_comprehension_context_arg}, + comprehension_clause{comprehension_clause_arg}, + comprehension_clause_loc{comprehension_clause_loc_arg}, + comprehension_start_condition{comprehension_start_condition_arg}, + comprehension_iter_nb{comprehension_iter_nb_arg} { } }; @@ -114,6 +129,13 @@ private: //! Set to true while parsing an IF statement (only the statement, not the body) bool reading_if_statement; + bool is_comprehension_context{false}; + int comprehension_iter_nb{0}; + int comprehension_start_condition; + int nested_comprehension_nb{0}; + string comprehension_clause, comprehension_clause_tmp; + Macro::parser::location_type comprehension_clause_loc, comprehension_clause_loc_tmp; + //! Output the @#line declaration void output_line(Macro::parser::location_type *yylloc) const; @@ -141,6 +163,9 @@ private: //! Initialise a new flex buffer with the loop body void new_loop_body_buffer(Macro::parser::location_type *yylloc); + //! Initialize a new flex buffer with the comprehension conditional clause + void new_comprehension_clause_buffer(Macro::parser::location_type *yylloc); + public: MacroFlex(istream *in, ostream *out, bool no_line_macro_arg, vector path_arg); @@ -169,6 +194,8 @@ private: //! Second is the array over which iteration is done //! Third is subscript to be used by next call of iter_loop() (beginning with 0) */ stack, shared_ptr, int>> loop_stack; + + stack, shared_ptr, int>> comprehension_stack; public: //! Exception thrown when value of an unknown variable is requested class UnknownVariable @@ -239,6 +266,11 @@ public: in that case it destroys the pointer given to init_loop() */ bool iter_loop() noexcept(false); + void init_comprehension(const vector &names, MacroValuePtr value); + void iter_comprehension(); + void possibly_add_comprehension_element(vector &v, MacroValuePtr test_expr) const; + int get_comprehension_iter_nb() const; + //! Begins an @#if statement void begin_if(const MacroValuePtr &value) noexcept(false); diff --git a/src/macro/MacroFlex.ll b/src/macro/MacroFlex.ll index 5dee6b33..0de931cd 100644 --- a/src/macro/MacroFlex.ll +++ b/src/macro/MacroFlex.ll @@ -55,6 +55,7 @@ using token = Macro::parser::token; %x FOR_BODY %x THEN_BODY %x ELSE_BODY +%x COMPREHENSION_CLAUSE %{ // Increments location counter for every token read @@ -238,6 +239,17 @@ CONT \\\\ yylval->build(yytext + 1).pop_back(); return token::STRING; } +; { + comprehension_clause_tmp.erase(); + nested_comprehension_nb = 0; + // Save start condition (either STMT or EXPR) + comprehension_start_condition = YY_START; + // Save location + comprehension_clause_loc_tmp = *yylloc; + BEGIN(COMPREHENSION_CLAUSE); + return token::SEMICOLON; + } +@ { return token::ATSIGN; } // Used for separation of repeated comprehension clauses line { return token::LINE; } define { return token::DEFINE; } @@ -263,7 +275,22 @@ CONT \\\\ return token::NAME; } -<> { driver.error(*yylloc, "Unexpected end of file while parsing a macro expression"); } +<> { + if (!is_comprehension_context) + driver.error(*yylloc, "Unexpected end of file while parsing a macro expression"); + else + { + if (--comprehension_iter_nb > 0) + new_comprehension_clause_buffer(yylloc); + else + { + restore_context(yylloc); + + BEGIN(comprehension_start_condition); + return token::RBRACKET; + } + } + } <> { driver.error(*yylloc, "Unexpected end of file while parsing a macro statement"); } {EOL} { yylloc->lines(1); yylloc->step(); for_body_tmp.append(yytext); } @@ -385,6 +412,39 @@ CONT \\\\ } } +{EOL} { driver.error(*yylloc, "Unexpected line break in comprehension"); } +<> { driver.error(*yylloc, "Unexpected end of file in comprehension"); } +[^\[\]] { comprehension_clause_tmp.append(yytext); yylloc->step(); } +\[ { nested_comprehension_nb++; comprehension_clause_tmp.append(yytext); yylloc->step(); } +\] { + yylloc->step(); + if (nested_comprehension_nb) + { + nested_comprehension_nb--; + comprehension_clause_tmp.append(yytext); + } + else + { + int comprehension_iter_nb_tmp = driver.get_comprehension_iter_nb(); + comprehension_clause_tmp.append(" @ "); + + if (comprehension_iter_nb_tmp > 0) + { + // Save old buffer state and location + save_context(yylloc); + + is_comprehension_context = true; + comprehension_iter_nb = comprehension_iter_nb_tmp; + comprehension_clause = comprehension_clause_tmp; + comprehension_clause_loc = comprehension_clause_loc_tmp; + + new_comprehension_clause_buffer(yylloc); + } + + BEGIN(EXPR); + } + } + <> { // Quit lexer if end of main file if (context_stack.empty()) @@ -444,7 +504,9 @@ void MacroFlex::save_context(Macro::parser::location_type *yylloc) { context_stack.push(ScanContext(input, YY_CURRENT_BUFFER, *yylloc, is_for_context, - for_body, for_body_loc)); + for_body, for_body_loc, is_comprehension_context, + comprehension_clause, comprehension_clause_loc, + comprehension_start_condition, comprehension_iter_nb)); } void @@ -456,10 +518,15 @@ MacroFlex::restore_context(Macro::parser::location_type *yylloc) is_for_context = context_stack.top().is_for_context; for_body = context_stack.top().for_body; for_body_loc = context_stack.top().for_body_loc; + if (!is_comprehension_context) + output_line(yylloc); // Dump @#line instruction + is_comprehension_context = context_stack.top().is_comprehension_context; + comprehension_clause = context_stack.top().comprehension_clause; + comprehension_clause_loc = context_stack.top().comprehension_clause_loc; + comprehension_start_condition = context_stack.top().comprehension_start_condition; + comprehension_iter_nb = context_stack.top().comprehension_iter_nb; // Remove top of stack context_stack.pop(); - // Dump @#line instruction - output_line(yylloc); } void @@ -556,6 +623,17 @@ MacroFlex::new_loop_body_buffer(Macro::parser::location_type *yylloc) yy_switch_to_buffer(yy_create_buffer(input, YY_BUF_SIZE)); } +void +MacroFlex::new_comprehension_clause_buffer(Macro::parser::location_type *yylloc) +{ + input = new stringstream(comprehension_clause); + *yylloc = comprehension_clause_loc; + yylloc->begin.filename = yylloc->end.filename = new string(*comprehension_clause_loc.begin.filename); + is_for_context = false; + for_body.clear(); + yy_switch_to_buffer(yy_create_buffer(input, YY_BUF_SIZE)); +} + /* This implementation of MacroFlexLexer::yylex() is required to fill the * vtable of the class MacroFlexLexer. We define the scanner's main yylex * function via YY_DECL to reside in the MacroFlex class instead. */