stepan
/
preprocessor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert "preprocessor: add utf-8 tokenizing" 

This reverts commit f2cc9d389954f16dbb88c3401e90c7981e3599d6.

This commit causes warnings in Flex
issue#70
Houtan Bastani 2015-10-30 18:27:41 +01:00
parent 110823e2c7
commit 035b6ea463
2 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
DynareFlex.ll
View File

@ -787,7 +787,7 @@ DATE -?[0-9]+([YyAa]|[Mm]([1-9]|1[0-2])|[Qq][1-4]|[Ww]([1-9]{1}|[1-4][0-9]|5[0-2
<DYNARE_STATEMENT>max_dim_cova_group {return token::MAX_DIM_COVA_GROUP;} <DYNARE_STATEMENT>max_dim_cova_group {return token::MAX_DIM_COVA_GROUP;}
<DYNARE_STATEMENT>gsa_sample_file {return token::GSA_SAMPLE_FILE;} <DYNARE_STATEMENT>gsa_sample_file {return token::GSA_SAMPLE_FILE;}
<DYNARE_STATEMENT,DYNARE_BLOCK>[A-Za-z_\x80-\xf3][A-Za-z0-9_\x80-\xf3]* { <DYNARE_STATEMENT,DYNARE_BLOCK>[A-Za-z_][A-Za-z0-9_]* {
yylval->string_val = new string(yytext); yylval->string_val = new string(yytext);
return token::NAME; return token::NAME;
} }
@ -849,7 +849,7 @@ DATE -?[0-9]+([YyAa]|[Mm]([1-9]|1[0-2])|[Qq][1-4]|[Ww]([1-9]{1}|[1-4][0-9]|5[0-2
element in initval (in which case Dynare recognizes the matrix name as an external element in initval (in which case Dynare recognizes the matrix name as an external
function symbol), and may want to modify the matrix later with Matlab statements. function symbol), and may want to modify the matrix later with Matlab statements.
*/ */
<INITIAL>[A-Za-z_\x80-\xf3][A-Za-z0-9_\x80-\xf3]* { <INITIAL>[A-Za-z_][A-Za-z0-9_]* {
if (driver.symbol_exists_and_is_not_modfile_local_or_external_function(yytext)) if (driver.symbol_exists_and_is_not_modfile_local_or_external_function(yytext))
{ {
BEGIN DYNARE_STATEMENT; BEGIN DYNARE_STATEMENT;
@ -867,7 +867,7 @@ DATE -?[0-9]+([YyAa]|[Mm]([1-9]|1[0-2])|[Qq][1-4]|[Ww]([1-9]{1}|[1-4][0-9]|5[0-2
/* For joint prior statement, match [symbol, symbol, ...] /* For joint prior statement, match [symbol, symbol, ...]
If no match, begin native and push everything back on stack If no match, begin native and push everything back on stack
*/ */
<INITIAL>\[([[:space:]]*[A-Za-z_\x80-\xf3][A-Za-z0-9_\x80-\xf3]*[[:space:]]*,{1}[[:space:]]*)*([[:space:]]*[A-Za-z_\x80-\xf3][A-Za-z0-9_\x80-\xf3]*[[:space:]]*){1}\] { <INITIAL>\[([[:space:]]*[A-Za-z_][A-Za-z0-9_]*[[:space:]]*,{1}[[:space:]]*)*([[:space:]]*[A-Za-z_][A-Za-z0-9_]*[[:space:]]*){1}\] {
string yytextcpy = string(yytext); string yytextcpy = string(yytext);
yytextcpy.erase(remove(yytextcpy.begin(), yytextcpy.end(), '['), yytextcpy.end()); yytextcpy.erase(remove(yytextcpy.begin(), yytextcpy.end(), '['), yytextcpy.end());
yytextcpy.erase(remove(yytextcpy.begin(), yytextcpy.end(), ']'), yytextcpy.end()); yytextcpy.erase(remove(yytextcpy.begin(), yytextcpy.end(), ']'), yytextcpy.end());

2
macro/MacroFlex.ll
View File

@ -245,7 +245,7 @@ CONT \\\\
<STMT>echo { return token::ECHO_DIR; } <STMT>echo { return token::ECHO_DIR; }
<STMT>error { return token::ERROR; } <STMT>error { return token::ERROR; }
<STMT,EXPR>[A-Za-z_\x80-\xf3][A-Za-z0-9_\x80-\xf3]* { <STMT,EXPR>[A-Za-z_][A-Za-z0-9_]* {
yylval->string_val = new string(yytext); yylval->string_val = new string(yytext);
return token::NAME; return token::NAME;
} }