llvm-journey

LLVM Journey
git clone git://0xff.ir/g/llvm-journey.git
Log | Files | Refs | README | LICENSE

kaleidoscope_lexer.hpp (1608B)

      1 
      2 #pragma once
      3 
      4 #include <algorithm>
      5 #include <cctype>
      6 #include <iterator>
      7 #include <string>
      8 
      9 namespace kal {
     10 
     11 enum class TkType
     12 {
     13   END,
     14   Def,
     15   Extern,
     16   Comment,
     17   Id,
     18   Num,
     19   Etc,
     20 };
     21 static const std::string TKTYPE_STR[] = {
     22   "END", "Def", "Extern", "Comment", "Id", "Num", "Etc",
     23 };
     24 
     25 // TODO token(TkType, FwdIt tkBegin, FwdIt tkEnd, size_t line, size_t col)
     26 // token(TkType, FwdIt tkBegin, FwdIt tkEnd)
     27 template<typename FwdIt, typename TokenOp>
     28 FwdIt
     29 lex(FwdIt f, FwdIt l, TokenOp token)
     30 {
     31   static const std::string sDef{ "def" };
     32   static const std::string sExtern{ "extern" };
     33   FwdIt b; // begin
     34   FwdIt e; // end
     35 
     36   while (f != l) {
     37     while (f != l && isspace(*f))
     38       ++f;
     39 
     40     if (f == l)
     41       break;
     42 
     43     if (isalpha(*f)) {
     44       b = f++;
     45       while (f != l && isalnum(*f))
     46         ++f;
     47       e = f;
     48 
     49       if (std::equal(b, e, std::cbegin(sDef), std::cend(sDef)))
     50         token(TkType::Def, b, e);
     51       else if (std::equal(b, e, std::begin(sExtern), std::end(sExtern)))
     52         token(TkType::Extern, b, e);
     53       else // [a-zA-Z][a-zA-Z0-9]*
     54         token(TkType::Id, b, e);
     55     } else if (isdigit(*f) || *f == '.') { // Num: [0-9.]+ // FIXME
     56       b = f++;
     57       while (f != l && (isdigit(*f) || *f == '.'))
     58         ++f;
     59       e = f;
     60 
     61       token(TkType::Num, b, e);
     62     } else if (*f == '#') { // Comment
     63       b = f++;
     64       while (f != l && (*f != '\n' && *f != '\r'))
     65         ++f;
     66       e = f;
     67 
     68       token(TkType::Comment, b, e);
     69     } else {
     70       b = f++;
     71       e = f;
     72 
     73       token(TkType::Etc, b, e);
     74     }
     75   }
     76 
     77   token(TkType::END, f, f);
     78   return f;
     79 }
     80 
     81 } // namespace kal