From 035b6ea46397ec1c76aa1ce48886e6ceb3fe5128 Mon Sep 17 00:00:00 2001
From: Houtan Bastani <houtan@dynare.org>
Date: Fri, 30 Oct 2015 18:27:41 +0100
Subject: [PATCH] Revert "preprocessor: add utf-8 tokenizing"

This reverts commit f2cc9d389954f16dbb88c3401e90c7981e3599d6.

This commit causes warnings in Flex
---
 DynareFlex.ll      | 6 +++---
 macro/MacroFlex.ll | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/DynareFlex.ll b/DynareFlex.ll
index 0686500d..152b3962 100644
--- a/DynareFlex.ll
+++ b/DynareFlex.ll
@@ -787,7 +787,7 @@ DATE -?[0-9]+([YyAa]|[Mm]([1-9]|1[0-2])|[Qq][1-4]|[Ww]([1-9]{1}|[1-4][0-9]|5[0-2
 <DYNARE_STATEMENT>max_dim_cova_group {return token::MAX_DIM_COVA_GROUP;}
 <DYNARE_STATEMENT>gsa_sample_file {return token::GSA_SAMPLE_FILE;}
 
-<DYNARE_STATEMENT,DYNARE_BLOCK>[A-Za-z_\x80-\xf3][A-Za-z0-9_\x80-\xf3]* {
+<DYNARE_STATEMENT,DYNARE_BLOCK>[A-Za-z_][A-Za-z0-9_]* {
   yylval->string_val = new string(yytext);
   return token::NAME;
 }
@@ -849,7 +849,7 @@ DATE -?[0-9]+([YyAa]|[Mm]([1-9]|1[0-2])|[Qq][1-4]|[Ww]([1-9]{1}|[1-4][0-9]|5[0-2
     element in initval (in which case Dynare recognizes the matrix name as an external
     function symbol), and may want to modify the matrix later with Matlab statements.
  */
-<INITIAL>[A-Za-z_\x80-\xf3][A-Za-z0-9_\x80-\xf3]* {
+<INITIAL>[A-Za-z_][A-Za-z0-9_]* {
   if (driver.symbol_exists_and_is_not_modfile_local_or_external_function(yytext))
     {
       BEGIN DYNARE_STATEMENT;
@@ -867,7 +867,7 @@ DATE -?[0-9]+([YyAa]|[Mm]([1-9]|1[0-2])|[Qq][1-4]|[Ww]([1-9]{1}|[1-4][0-9]|5[0-2
  /* For joint prior statement, match [symbol, symbol, ...]
    If no match, begin native and push everything back on stack
  */
-<INITIAL>\[([[:space:]]*[A-Za-z_\x80-\xf3][A-Za-z0-9_\x80-\xf3]*[[:space:]]*,{1}[[:space:]]*)*([[:space:]]*[A-Za-z_\x80-\xf3][A-Za-z0-9_\x80-\xf3]*[[:space:]]*){1}\] {
+<INITIAL>\[([[:space:]]*[A-Za-z_][A-Za-z0-9_]*[[:space:]]*,{1}[[:space:]]*)*([[:space:]]*[A-Za-z_][A-Za-z0-9_]*[[:space:]]*){1}\] {
   string yytextcpy = string(yytext);
   yytextcpy.erase(remove(yytextcpy.begin(), yytextcpy.end(), '['), yytextcpy.end());
   yytextcpy.erase(remove(yytextcpy.begin(), yytextcpy.end(), ']'), yytextcpy.end());
diff --git a/macro/MacroFlex.ll b/macro/MacroFlex.ll
index 9229b049..9e694eb4 100644
--- a/macro/MacroFlex.ll
+++ b/macro/MacroFlex.ll
@@ -245,7 +245,7 @@ CONT \\\\
 <STMT>echo                  { return token::ECHO_DIR; }
 <STMT>error                 { return token::ERROR; }
 
-<STMT,EXPR>[A-Za-z_\x80-\xf3][A-Za-z0-9_\x80-\xf3]* {
+<STMT,EXPR>[A-Za-z_][A-Za-z0-9_]* {
                               yylval->string_val = new string(yytext);
                               return token::NAME;
                             }