From 404826e78a56e15e20d3938aed80945295921745 Mon Sep 17 00:00:00 2001
From: HombreLaser <buran@silosneeded.com>
Date: Sat, 22 Jun 2024 16:45:04 -0600
Subject: Add parser

---
 src/tokenizer.cpp | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

(limited to 'src/tokenizer.cpp')

diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index a156000..783803e 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -1,11 +1,11 @@
 #include <string>
-#include <array>
 #include "include/tokenizer.hpp"
 #include "exceptions/tokenizer_exception.hpp"
 
 Tokenizer::Tokenizer() {
-  tokens.fill(nullptr);
   tokens_head = 0;
+  for(int i = 0; i < TOKENS_ARRAY_LEN; ++i)
+    tokens[i] = nullptr;
 }
 
 Tokenizer::~Tokenizer() {
@@ -13,10 +13,14 @@ Tokenizer::~Tokenizer() {
 }
 
 void Tokenizer::clearTokens() {
-  for(auto token{ tokens.begin() }; *token != nullptr; ++token)
-    delete token;
-
-  tokens.fill(nullptr);
+  tokens_head = 0;
+  
+  for(int i = 0; i < TOKENS_ARRAY_LEN; ++i) {
+    if(tokens[i] != nullptr) {
+      delete tokens[i];
+      tokens[i] = nullptr;
+    }
+  }
 }
 
 void Tokenizer::insertToken(Type type, int value = 0) {
@@ -27,8 +31,8 @@ void Tokenizer::insertToken(Type type, int value = 0) {
 }
 
 
-const std::array<Token *, 16>
-&Tokenizer::tokenize(const std::string &operation) {
+Token **Tokenizer::tokenize(const std::string &operation) {
+  clearTokens();
   to_tokenize = &operation;
   current_char_index = 0;
   
@@ -44,6 +48,8 @@ const std::array<Token *, 16>
       throw TokenizerException("Invalid character detected.");
   }
 
+  // "Nil" terminated array. Intended to be the epsilon terminal in the grammar.
+  tokens[tokens_head] = new Token { .type = nil, .value = 0 };
   return getTokens();
 }
 
@@ -72,7 +78,7 @@ void Tokenizer::matchParens(const std::string &operation) {
 
   std::string remaining(operation.substr(current_char_index)); 
   bool result = std::regex_search(remaining, match,
-				  operand_regex);
+				  parens_regex);
 
   if(!result)
     return;
@@ -97,7 +103,7 @@ void Tokenizer::matchOperation(const std::string &operation) {
 
   std::string remaining(operation.substr(current_char_index)); 
   bool result = std::regex_search(remaining, match,
-				  operand_regex);
+				  operation_regex);
 
   if(!result)
     return;
@@ -120,6 +126,6 @@ void Tokenizer::matchOperation(const std::string &operation) {
   current_char_index += 1;
 }
 
-const std::array<Token *, 16> &Tokenizer::getTokens() {
+Token **Tokenizer::getTokens() {
   return tokens;
 }
-- 
cgit v1.2.3