llvm-journey

LLVM Journey
git clone git://0xff.ir/g/llvm-journey.git
Log | Files | Refs | README | LICENSE

kaleidoscope_tokens.hpp (1899B)

      1 
      2 #pragma once
      3 
      4 #include "kaleidoscope_lexer.hpp"
      5 
      6 #include <cstddef>
      7 #include <sstream>
      8 #include <string>
      9 #include <type_traits>
     10 #include <utility>
     11 #include <vector>
     12 
     13 namespace kal {
     14 
     15 struct Token
     16 {
     17   TkType type;
     18   ptrdiff_t bpos;
     19   ptrdiff_t epos;
     20   std::string str;
     21   double dbl;
     22 
     23   friend bool operator==(const Token& x, const Token& y)
     24   {
     25     return x.type == y.type && x.bpos == y.bpos && x.epos == y.epos &&
     26            x.str == y.str && /*FIXME floating-point eq*/ x.dbl == y.dbl;
     27   }
     28 
     29   friend bool operator!=(const Token& x, const Token& y) { return !(x == y); }
     30 };
     31 
     32 std::string
     33 to_string(const Token& t)
     34 {
     35   std::ostringstream oss;
     36 
     37   // clang-format off
     38 
     39   oss << "Token{"
     40       << "\n  kal::TkType::" << kal::TKTYPE_STR[static_cast<int>(t.type)]
     41       << ",\n  " << t.bpos
     42       << ",\n  " << t.epos
     43       << ",\n  \"" << t.str << '"'
     44       << ",\n  " << t.dbl
     45       << "\n}\n";
     46 
     47   // clang-format on
     48 
     49   return oss.str();
     50 }
     51 
     52 inline TkType
     53 token_type(const Token& t)
     54 {
     55   return t.type;
     56 }
     57 
     58 inline std::string
     59 token_str(const Token& t)
     60 {
     61   return t.str;
     62 }
     63 
     64 inline double
     65 token_num(const Token& t)
     66 {
     67   return t.dbl;
     68 }
     69 
     70 inline int
     71 token_precedence(const Token& t)
     72 {
     73   if (t.type != TkType::Etc)
     74     return -1;
     75 
     76   // assert(!t.str.empty());
     77 
     78   switch (t.str[0]) {
     79     case '<':
     80       return 10;
     81 
     82     case '+':
     83     case '-':
     84       return 20;
     85 
     86     case '*':
     87     case '/':
     88       return 40;
     89   }
     90 
     91   return -1;
     92 }
     93 
     94 template<typename FwdIt, typename OutIt>
     95 OutIt
     96 tokenize(FwdIt first, FwdIt last, OutIt tokens)
     97 {
     98   static_assert(std::is_assignable<decltype(*tokens), kal::Token>::value, "");
     99 
    100   kal::lex(first, last, [&](auto t, auto f, auto l) {
    101     auto b = f - first;
    102     auto e = l - first;
    103     auto tk = kal::Token{ t, b, e, std::string{ f, l }, 0 };
    104 
    105     if (t == kal::TkType::Num)
    106       tk.dbl = std::stod(tk.str);
    107 
    108     *tokens++ = std::move(tk);
    109   });
    110 
    111   return tokens;
    112 }
    113 
    114 } // namespace kal