llvm-journey

commit bb0d2b3952c3427deeb44fcdeb4f42d13a5c25cd
parent e323acc67f313d3a5ecf7b9f23b46f550b5394c1
Author: Mohammad-Reza Nabipoor <m.nabipoor@yahoo.com>
Date:   Wed, 12 Aug 2020 22:38:50 +0430

Add kal::Token data type & update tests

Diffstat:
M
Makefile
 | 
4
++--
A
kaleidoscope_tokens.hpp
 | 
51
+++++++++++++++++++++++++++++++++++++++++++++++++++
M
tests/kaleidoscope_lexer.test.cpp
 | 
22
++++------------------

3 files changed, 57 insertions(+), 20 deletions(-)
diff --git a/Makefile b/Makefile
@@ -30,8 +30,8 @@ kaleidoscope_codegen.o: kaleidoscope_codegen.cpp kaleidoscope_codegen.hpp \
 #--- tests
 
 kaleidoscope_lexer.test.o: CXXFLAGS += $(tflags)
-kaleidoscope_lexer.test.o: kaleidoscope_lexer.hpp $(catch2)
-kaleidoscope_lexer.test.o: tests/kaleidoscope_lexer.test.cpp
+kaleidoscope_lexer.test.o: kaleidoscope_lexer.hpp kaleidoscope_tokens.hpp
+kaleidoscope_lexer.test.o: tests/kaleidoscope_lexer.test.cpp $(catch2)
 	$(CXX) $(CXXFLAGS) -c -o $@ $<
 kaleidoscope_lexer.test: kaleidoscope_lexer.test.o
 	$(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS)
diff --git a/kaleidoscope_tokens.hpp b/kaleidoscope_tokens.hpp
@@ -0,0 +1,51 @@
+
+#pragma once
+
+#include "kaleidoscope_lexer.hpp"
+
+#include <cstddef>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+namespace kal {
+
+struct Token
+{
+  kal::TkType type;
+  ptrdiff_t bpos;
+  ptrdiff_t epos;
+  std::string str;
+  double dbl;
+
+  friend bool operator==(const Token& x, const Token& y)
+  {
+    return x.type == y.type && x.bpos == y.bpos && x.epos == y.epos &&
+           x.str == y.str && /*FIXME floating-point eq*/ x.dbl == y.dbl;
+  }
+
+  friend bool operator!=(const Token& x, const Token& y) { return !(x == y); }
+};
+
+template<typename FwdIt, typename OutIt>
+OutIt
+tokenize(FwdIt first, FwdIt last, OutIt tokens)
+{
+  static_assert(std::is_assignable<decltype(*tokens), kal::Token>::value, "");
+
+  kal::lex(first, last, [&](auto t, auto f, auto l) {
+    auto b = f - first;
+    auto e = l - first;
+    auto tk = kal::Token{ t, b, e, std::string{ f, l }, 0 };
+
+    if (t == kal::TkType::Num)
+      tk.dbl = std::stod(tk.str);
+
+    *tokens++ = std::move(tk);
+  });
+
+  return tokens;
+}
+
+} // namespace kal
diff --git a/tests/kaleidoscope_lexer.test.cpp b/tests/kaleidoscope_lexer.test.cpp
@@ -8,22 +8,7 @@
 #include <string>
 #include <vector>
 
-#include <iostream> // FIXME
-
-struct Token
-{
-  kal::TkType type;
-  ptrdiff_t bpos;
-  ptrdiff_t epos;
-  std::string str;
-  double dbl;
-
-  friend bool operator==(const Token& x, const Token& y)
-  {
-    return x.type == y.type && x.bpos == y.bpos && x.epos == y.epos &&
-           x.str == y.str && /*FIXME floating-point eq*/ x.dbl == y.dbl;
-  }
-};
+#include "kaleidoscope_tokens.hpp"
 
 TEST_CASE("lex", "[lex]")
 {
@@ -39,15 +24,16 @@ fib(40)
 
 extern sin(arg);
 )" };
-  std::vector<Token> tokens;
+  std::vector<kal::Token> tokens;
   auto sbeg = std::cbegin(src);
   auto send = std::cend(src);
+  using kal::Token;
 
   auto last =
     kal::lex(sbeg, send, [&](kal::TkType t, const auto& f, const auto& l) {
       auto b = f - sbeg;
       auto e = l - sbeg;
-      auto tk = Token{ t, b, e, src.substr(b, e - b), 0 };
+      auto tk = kal::Token{ t, b, e, src.substr(b, e - b), 0 };
 
       if (t == kal::TkType::Num)
         tk.dbl = std::stod(tk.str);