stepan
/
preprocessor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Macro-processor: implement comprehensions 

Due to a limitation of the current implementation, this breaks syntaxes like
[ (i,j) ] (but not [ (2,j) ]; the problem only occurs when an array is
constructed by specifying as first element a tuple whose first element is a
variable name). Solving this problem requires an overhaul of the
macro-processor, with construction of ASTs at parsing time, and evaluation
later on (instead of doing on-the-fly evaluation).

Ref #5
issue#70
Sébastien Villemot 2018-09-24 17:57:55 +02:00
parent 7a5cc7e54b
commit 3e5c8dd80d
4 changed files with 226 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
src/macro/MacroBison.yy
View File

@ -68,6 +68,7 @@ class MacroDriver;
%token COMMA DEFINE LINE FOR IN IF ECHO_DIR ERROR IFDEF IFNDEF POWER
%token LPAREN RPAREN LBRACKET RBRACKET EQUAL EOL LENGTH ECHOMACROVARS SAVE
%token SEMICOLON ATSIGN
%token <int> INTEGER
%token <string> NAME STRING
@ -88,7 +89,7 @@ class MacroDriver;
%type <vector<string>> comma_name
%type <MacroValuePtr> expr
%type <vector<MacroValuePtr>> comma_expr tuple_comma_expr
%type <vector<MacroValuePtr>> comma_expr tuple_comma_expr comprehension_clause
%%
%start statement_list_or_nothing;
@ -217,6 +218,24 @@ expr : INTEGER
{ TYPERR_CATCH($$ = $1->set_intersection($3), @$); }
| expr POWER expr
{ TYPERR_CATCH($$ = $1->power($3), @$); }
| LBRACKET NAME IN expr SEMICOLON
{
driver.init_comprehension(vector<string>{$2}, $4);
driver.iter_comprehension();
}
comprehension_clause RBRACKET
{
$$ = make_shared<ArrayMV>($7);
}
| LBRACKET LPAREN comma_name RPAREN IN expr SEMICOLON
{
driver.init_comprehension($3, $6);
driver.iter_comprehension();
}
comprehension_clause RBRACKET
{
$$ = make_shared<ArrayMV>($9);
}
;
comma_expr : %empty
@ -237,6 +256,23 @@ tuple_comma_expr : %empty
{ $1.push_back($3); $$ = $1; }
;
/* The lexer will repeat the comprehension clause as many times as there are
elements in the set to be filtered. It also adds a dummy at-sign (@) at the
end of every repetition (for making parsing of repetitions unambiguous). */
comprehension_clause : expr ATSIGN
{
$$ = vector<MacroValuePtr>{};
driver.possibly_add_comprehension_element($$, $1);
driver.iter_comprehension();
}
| comprehension_clause expr ATSIGN
{
$$ = $1;
driver.possibly_add_comprehension_element($$, $2);
driver.iter_comprehension();
}
;
%%
void

73
src/macro/MacroDriver.cc
View File

@ -260,6 +260,79 @@ MacroDriver::iter_loop() noexcept(false)
}
}
void
MacroDriver::init_comprehension(const vector<string> &names, MacroValuePtr value)
{
auto mv = dynamic_pointer_cast<ArrayMV>(value);
if (!mv)
throw MacroValue::TypeError("In a comprehension, the expression after the 'in' keyword must be an array");
comprehension_stack.emplace(names, move(mv), 0);
}
int
MacroDriver::get_comprehension_iter_nb() const
{
assert(!comprehension_stack.empty());
auto &mv = get<1>(comprehension_stack.top());
return mv->values.size();
}
void
MacroDriver::iter_comprehension()
{
assert(!comprehension_stack.empty());
int &i = get<2>(comprehension_stack.top());
auto &mv = get<1>(comprehension_stack.top());
vector<string> &names = get<0>(comprehension_stack.top());
assert(i <= static_cast<int>(mv->values.size()));
if (i == static_cast<int>(mv->values.size()))
comprehension_stack.pop();
else
{
if (names.size() == 1)
env[names.at(0)] = mv->values[i++];
else
{
auto tmv = dynamic_pointer_cast<TupleMV>(mv->values[i++]);
if (!tmv)
throw MacroValue::TypeError("The expression after the 'in' keyword must be an array expression of tuples");
if (tmv->values.size() != names.size())
{
cerr << "Error in comprehension loop: tuple in array contains " << tmv->length()
<< " elements while you are assigning to " << names.size() << " variables."
<< endl;
exit(EXIT_FAILURE);
}
for (auto &name: names)
{
auto idx = &name - &names[0];
env[name] = tmv->values.at(idx);
}
}
}
}
void
MacroDriver::possibly_add_comprehension_element(vector<MacroValuePtr> &v, MacroValuePtr test_expr) const
{
auto ival = dynamic_pointer_cast<IntMV>(test_expr);
if (!ival)
throw MacroValue::TypeError("In a comprehension, the expression after the 'if' must evaluate to an integer");
if (ival->value)
{
assert(!comprehension_stack.empty());
const int &i = get<2>(comprehension_stack.top());
auto &mv = get<1>(comprehension_stack.top());
v.push_back(mv->values.at(i-1));
}
}
void
MacroDriver::begin_if(const MacroValuePtr &value) noexcept(false)
{

36
src/macro/MacroDriver.hh
View File

@ -60,12 +60,27 @@ private:
const bool is_for_context;
const string for_body;
const Macro::parser::location_type for_body_loc;
const bool is_comprehension_context;
const string comprehension_clause;
const Macro::parser::location_type comprehension_clause_loc;
const int comprehension_start_condition;
const int comprehension_iter_nb;
ScanContext(istream *input_arg, struct yy_buffer_state *buffer_arg,
Macro::parser::location_type &yylloc_arg, bool is_for_context_arg,
string for_body_arg,
Macro::parser::location_type &for_body_loc_arg) :
Macro::parser::location_type &for_body_loc_arg,
bool is_comprehension_context_arg,
string comprehension_clause_arg,
Macro::parser::location_type &comprehension_clause_loc_arg,
int comprehension_start_condition_arg,
int comprehension_iter_nb_arg) :
input(input_arg), buffer(buffer_arg), yylloc(yylloc_arg), is_for_context(is_for_context_arg),
for_body(move(for_body_arg)), for_body_loc(for_body_loc_arg)
for_body(move(for_body_arg)), for_body_loc(for_body_loc_arg),
is_comprehension_context{is_comprehension_context_arg},
comprehension_clause{comprehension_clause_arg},
comprehension_clause_loc{comprehension_clause_loc_arg},
comprehension_start_condition{comprehension_start_condition_arg},
comprehension_iter_nb{comprehension_iter_nb_arg}
{
}
};
@ -114,6 +129,13 @@ private:
//! Set to true while parsing an IF statement (only the statement, not the body)
bool reading_if_statement;
bool is_comprehension_context{false};
int comprehension_iter_nb{0};
int comprehension_start_condition;
int nested_comprehension_nb{0};
string comprehension_clause, comprehension_clause_tmp;
Macro::parser::location_type comprehension_clause_loc, comprehension_clause_loc_tmp;
//! Output the @#line declaration
void output_line(Macro::parser::location_type *yylloc) const;
@ -141,6 +163,9 @@ private:
//! Initialise a new flex buffer with the loop body
void new_loop_body_buffer(Macro::parser::location_type *yylloc);
//! Initialize a new flex buffer with the comprehension conditional clause
void new_comprehension_clause_buffer(Macro::parser::location_type *yylloc);
public:
MacroFlex(istream *in, ostream *out, bool no_line_macro_arg, vector<string> path_arg);
@ -169,6 +194,8 @@ private:
//! Second is the array over which iteration is done
//! Third is subscript to be used by next call of iter_loop() (beginning with 0) */
stack<tuple<vector<string>, shared_ptr<ArrayMV>, int>> loop_stack;
stack<tuple<vector<string>, shared_ptr<ArrayMV>, int>> comprehension_stack;
public:
//! Exception thrown when value of an unknown variable is requested
class UnknownVariable
@ -239,6 +266,11 @@ public:
in that case it destroys the pointer given to init_loop() */
bool iter_loop() noexcept(false);
void init_comprehension(const vector<string> &names, MacroValuePtr value);
void iter_comprehension();
void possibly_add_comprehension_element(vector<MacroValuePtr> &v, MacroValuePtr test_expr) const;
int get_comprehension_iter_nb() const;
//! Begins an @#if statement
void begin_if(const MacroValuePtr &value) noexcept(false);

86
src/macro/MacroFlex.ll
View File

@ -55,6 +55,7 @@ using token = Macro::parser::token;
%x FOR_BODY
%x THEN_BODY
%x ELSE_BODY
%x COMPREHENSION_CLAUSE
%{
// Increments location counter for every token read
@ -238,6 +239,17 @@ CONT \\\\
yylval->build<string>(yytext + 1).pop_back();
return token::STRING;
}
<STMT,EXPR>; {
comprehension_clause_tmp.erase();
nested_comprehension_nb = 0;
// Save start condition (either STMT or EXPR)
comprehension_start_condition = YY_START;
// Save location
comprehension_clause_loc_tmp = *yylloc;
BEGIN(COMPREHENSION_CLAUSE);
return token::SEMICOLON;
}
<EXPR>@ { return token::ATSIGN; } // Used for separation of repeated comprehension clauses
<STMT>line { return token::LINE; }
<STMT>define { return token::DEFINE; }
@ -263,7 +275,22 @@ CONT \\\\
return token::NAME;
}
<EXPR><<EOF>> { driver.error(*yylloc, "Unexpected end of file while parsing a macro expression"); }
<EXPR><<EOF>> {
if (!is_comprehension_context)
driver.error(*yylloc, "Unexpected end of file while parsing a macro expression");
else
{
if (--comprehension_iter_nb > 0)
new_comprehension_clause_buffer(yylloc);
else
{
restore_context(yylloc);
BEGIN(comprehension_start_condition);
return token::RBRACKET;
}
}
}
<STMT><<EOF>> { driver.error(*yylloc, "Unexpected end of file while parsing a macro statement"); }
<FOR_BODY>{EOL} { yylloc->lines(1); yylloc->step(); for_body_tmp.append(yytext); }
@ -385,6 +412,39 @@ CONT \\\\
}
}
<COMPREHENSION_CLAUSE>{EOL} { driver.error(*yylloc, "Unexpected line break in comprehension"); }
<COMPREHENSION_CLAUSE><<EOF>> { driver.error(*yylloc, "Unexpected end of file in comprehension"); }
<COMPREHENSION_CLAUSE>[^\[\]] { comprehension_clause_tmp.append(yytext); yylloc->step(); }
<COMPREHENSION_CLAUSE>\[ { nested_comprehension_nb++; comprehension_clause_tmp.append(yytext); yylloc->step(); }
<COMPREHENSION_CLAUSE>\] {
yylloc->step();
if (nested_comprehension_nb)
{
nested_comprehension_nb--;
comprehension_clause_tmp.append(yytext);
}
else
{
int comprehension_iter_nb_tmp = driver.get_comprehension_iter_nb();
comprehension_clause_tmp.append(" @ ");
if (comprehension_iter_nb_tmp > 0)
{
// Save old buffer state and location
save_context(yylloc);
is_comprehension_context = true;
comprehension_iter_nb = comprehension_iter_nb_tmp;
comprehension_clause = comprehension_clause_tmp;
comprehension_clause_loc = comprehension_clause_loc_tmp;
new_comprehension_clause_buffer(yylloc);
}
BEGIN(EXPR);
}
}
<INITIAL><<EOF>> {
// Quit lexer if end of main file
if (context_stack.empty())
@ -444,7 +504,9 @@ void
MacroFlex::save_context(Macro::parser::location_type *yylloc)
{
context_stack.push(ScanContext(input, YY_CURRENT_BUFFER, *yylloc, is_for_context,
for_body, for_body_loc));
for_body, for_body_loc, is_comprehension_context,
comprehension_clause, comprehension_clause_loc,
comprehension_start_condition, comprehension_iter_nb));
}
void
@ -456,10 +518,15 @@ MacroFlex::restore_context(Macro::parser::location_type *yylloc)
is_for_context = context_stack.top().is_for_context;
for_body = context_stack.top().for_body;
for_body_loc = context_stack.top().for_body_loc;
if (!is_comprehension_context)
output_line(yylloc); // Dump @#line instruction
is_comprehension_context = context_stack.top().is_comprehension_context;
comprehension_clause = context_stack.top().comprehension_clause;
comprehension_clause_loc = context_stack.top().comprehension_clause_loc;
comprehension_start_condition = context_stack.top().comprehension_start_condition;
comprehension_iter_nb = context_stack.top().comprehension_iter_nb;
// Remove top of stack
context_stack.pop();
// Dump @#line instruction
output_line(yylloc);
}
void
@ -556,6 +623,17 @@ MacroFlex::new_loop_body_buffer(Macro::parser::location_type *yylloc)
yy_switch_to_buffer(yy_create_buffer(input, YY_BUF_SIZE));
}
void
MacroFlex::new_comprehension_clause_buffer(Macro::parser::location_type *yylloc)
{
input = new stringstream(comprehension_clause);
*yylloc = comprehension_clause_loc;
yylloc->begin.filename = yylloc->end.filename = new string(*comprehension_clause_loc.begin.filename);
is_for_context = false;
for_body.clear();
yy_switch_to_buffer(yy_create_buffer(input, YY_BUF_SIZE));
}
/* This implementation of MacroFlexLexer::yylex() is required to fill the
* vtable of the class MacroFlexLexer. We define the scanner's main yylex
* function via YY_DECL to reside in the MacroFlex class instead. */